diff options
Diffstat (limited to 'gi')
234 files changed, 0 insertions, 36886 deletions
diff --git a/gi/clda/src/Makefile.am b/gi/clda/src/Makefile.am deleted file mode 100644 index cdca1f97..00000000 --- a/gi/clda/src/Makefile.am +++ /dev/null @@ -1,6 +0,0 @@ -bin_PROGRAMS = clda - -clda_SOURCES = clda.cc - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -funroll-loops -I$(top_srcdir)/utils $(GTEST_CPPFLAGS) -AM_LDFLAGS = $(top_srcdir)/utils/libutils.a -lz diff --git a/gi/clda/src/ccrp.h b/gi/clda/src/ccrp.h deleted file mode 100644 index a7c2825c..00000000 --- a/gi/clda/src/ccrp.h +++ /dev/null @@ -1,291 +0,0 @@ -#ifndef _CCRP_H_ -#define _CCRP_H_ - -#include <numeric> -#include <cassert> -#include <cmath> -#include <list> -#include <iostream> -#include <vector> -#include <tr1/unordered_map> -#include <boost/functional/hash.hpp> -#include "sampler.h" -#include "slice_sampler.h" - -// Chinese restaurant process (Pitman-Yor parameters) with table tracking. - -template <typename Dish, typename DishHash = boost::hash<Dish> > -class CCRP { - public: - CCRP(double disc, double conc) : - num_tables_(), - num_customers_(), - discount_(disc), - concentration_(conc), - discount_prior_alpha_(std::numeric_limits<double>::quiet_NaN()), - discount_prior_beta_(std::numeric_limits<double>::quiet_NaN()), - concentration_prior_shape_(std::numeric_limits<double>::quiet_NaN()), - concentration_prior_rate_(std::numeric_limits<double>::quiet_NaN()) {} - - CCRP(double d_alpha, double d_beta, double c_shape, double c_rate, double d = 0.1, double c = 10.0) : - num_tables_(), - num_customers_(), - discount_(d), - concentration_(c), - discount_prior_alpha_(d_alpha), - discount_prior_beta_(d_beta), - concentration_prior_shape_(c_shape), - concentration_prior_rate_(c_rate) {} - - double discount() const { return discount_; } - double concentration() const { return concentration_; } - - bool has_discount_prior() const { - return !std::isnan(discount_prior_alpha_); - } - - bool has_concentration_prior() const { - return !std::isnan(concentration_prior_shape_); - } - - void clear() { - num_tables_ = 0; - num_customers_ = 0; - dish_locs_.clear(); - } - - unsigned num_tables(const Dish& dish) const { - const typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.find(dish); - if (it == dish_locs_.end()) return 0; - return it->second.table_counts_.size(); - } - - unsigned num_customers() const { - return num_customers_; - } - - unsigned num_customers(const Dish& dish) const { - const typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.find(dish); - if (it == dish_locs_.end()) return 0; - return it->total_dish_count_; - } - - // returns +1 or 0 indicating whether a new table was opened - int increment(const Dish& dish, const double& p0, MT19937* rng) { - DishLocations& loc = dish_locs_[dish]; - bool share_table = false; - if (loc.total_dish_count_) { - const double p_empty = (concentration_ + num_tables_ * discount_) * p0; - const double p_share = (loc.total_dish_count_ - loc.table_counts_.size() * discount_); - share_table = rng->SelectSample(p_empty, p_share); - } - if (share_table) { - double r = rng->next() * (loc.total_dish_count_ - loc.table_counts_.size() * discount_); - for (typename std::list<unsigned>::iterator ti = loc.table_counts_.begin(); - ti != loc.table_counts_.end(); ++ti) { - r -= (*ti - discount_); - if (r <= 0.0) { - ++(*ti); - break; - } - } - if (r > 0.0) { - std::cerr << "Serious error: r=" << r << std::endl; - Print(&std::cerr); - assert(r <= 0.0); - } - } else { - loc.table_counts_.push_back(1u); - ++num_tables_; - } - ++loc.total_dish_count_; - ++num_customers_; - return (share_table ? 0 : 1); - } - - // returns -1 or 0, indicating whether a table was closed - int decrement(const Dish& dish, MT19937* rng) { - DishLocations& loc = dish_locs_[dish]; - assert(loc.total_dish_count_); - if (loc.total_dish_count_ == 1) { - dish_locs_.erase(dish); - --num_tables_; - --num_customers_; - return -1; - } else { - int delta = 0; - // sample customer to remove UNIFORMLY. that is, do NOT use the discount - // here. if you do, it will introduce (unwanted) bias! - double r = rng->next() * loc.total_dish_count_; - --loc.total_dish_count_; - for (typename std::list<unsigned>::iterator ti = loc.table_counts_.begin(); - ti != loc.table_counts_.end(); ++ti) { - r -= *ti; - if (r <= 0.0) { - if ((--(*ti)) == 0) { - --num_tables_; - delta = -1; - loc.table_counts_.erase(ti); - } - break; - } - } - if (r > 0.0) { - std::cerr << "Serious error: r=" << r << std::endl; - Print(&std::cerr); - assert(r <= 0.0); - } - --num_customers_; - return delta; - } - } - - double prob(const Dish& dish, const double& p0) const { - const typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.find(dish); - const double r = num_tables_ * discount_ + concentration_; - if (it == dish_locs_.end()) { - return r * p0 / (num_customers_ + concentration_); - } else { - return (it->second.total_dish_count_ - discount_ * it->second.table_counts_.size() + r * p0) / - (num_customers_ + concentration_); - } - } - - double log_crp_prob() const { - return log_crp_prob(discount_, concentration_); - } - - static double log_beta_density(const double& x, const double& alpha, const double& beta) { - assert(x > 0.0); - assert(x < 1.0); - assert(alpha > 0.0); - assert(beta > 0.0); - const double lp = (alpha-1)*log(x)+(beta-1)*log(1-x)+lgamma(alpha+beta)-lgamma(alpha)-lgamma(beta); - return lp; - } - - static double log_gamma_density(const double& x, const double& shape, const double& rate) { - assert(x >= 0.0); - assert(shape > 0.0); - assert(rate > 0.0); - const double lp = (shape-1)*log(x) - shape*log(rate) - x/rate - lgamma(shape); - return lp; - } - - // taken from http://en.wikipedia.org/wiki/Chinese_restaurant_process - // does not include P_0's - double log_crp_prob(const double& discount, const double& concentration) const { - double lp = 0.0; - if (has_discount_prior()) - lp = log_beta_density(discount, discount_prior_alpha_, discount_prior_beta_); - if (has_concentration_prior()) - lp += log_gamma_density(concentration, concentration_prior_shape_, concentration_prior_rate_); - assert(lp <= 0.0); - if (num_customers_) { - if (discount > 0.0) { - const double r = lgamma(1.0 - discount); - lp += lgamma(concentration) - lgamma(concentration + num_customers_) - + num_tables_ * log(discount) + lgamma(concentration / discount + num_tables_) - - lgamma(concentration / discount); - assert(std::isfinite(lp)); - for (typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.begin(); - it != dish_locs_.end(); ++it) { - const DishLocations& cur = it->second; - for (std::list<unsigned>::const_iterator ti = cur.table_counts_.begin(); ti != cur.table_counts_.end(); ++ti) { - lp += lgamma(*ti - discount) - r; - } - } - } else { - assert(!"not implemented yet"); - } - } - assert(std::isfinite(lp)); - return lp; - } - - void resample_hyperparameters(MT19937* rng) { - assert(has_discount_prior() || has_concentration_prior()); - DiscountResampler dr(*this); - ConcentrationResampler cr(*this); - const int niterations = 10; - double gamma_upper = std::numeric_limits<double>::infinity(); - for (int iter = 0; iter < 5; ++iter) { - if (has_concentration_prior()) { - concentration_ = slice_sampler1d(cr, concentration_, *rng, 0.0, - gamma_upper, 0.0, niterations, 100*niterations); - } - if (has_discount_prior()) { - discount_ = slice_sampler1d(dr, discount_, *rng, std::numeric_limits<double>::min(), - 1.0, 0.0, niterations, 100*niterations); - } - } - concentration_ = slice_sampler1d(cr, concentration_, *rng, 0.0, - gamma_upper, 0.0, niterations, 100*niterations); - } - - struct DiscountResampler { - DiscountResampler(const CCRP& crp) : crp_(crp) {} - const CCRP& crp_; - double operator()(const double& proposed_discount) const { - return crp_.log_crp_prob(proposed_discount, crp_.concentration_); - } - }; - - struct ConcentrationResampler { - ConcentrationResampler(const CCRP& crp) : crp_(crp) {} - const CCRP& crp_; - double operator()(const double& proposed_concentration) const { - return crp_.log_crp_prob(crp_.discount_, proposed_concentration); - } - }; - - struct DishLocations { - DishLocations() : total_dish_count_() {} - unsigned total_dish_count_; // customers at all tables with this dish - std::list<unsigned> table_counts_; // list<> gives O(1) deletion and insertion, which we want - // .size() is the number of tables for this dish - }; - - void Print(std::ostream* out) const { - for (typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator it = dish_locs_.begin(); - it != dish_locs_.end(); ++it) { - (*out) << it->first << " (" << it->second.total_dish_count_ << " on " << it->second.table_counts_.size() << " tables): "; - for (typename std::list<unsigned>::const_iterator i = it->second.table_counts_.begin(); - i != it->second.table_counts_.end(); ++i) { - (*out) << " " << *i; - } - (*out) << std::endl; - } - } - - typedef typename std::tr1::unordered_map<Dish, DishLocations, DishHash>::const_iterator const_iterator; - const_iterator begin() const { - return dish_locs_.begin(); - } - const_iterator end() const { - return dish_locs_.end(); - } - - unsigned num_tables_; - unsigned num_customers_; - std::tr1::unordered_map<Dish, DishLocations, DishHash> dish_locs_; - - double discount_; - double concentration_; - - // optional beta prior on discount_ (NaN if no prior) - double discount_prior_alpha_; - double discount_prior_beta_; - - // optional gamma prior on concentration_ (NaN if no prior) - double concentration_prior_shape_; - double concentration_prior_rate_; -}; - -template <typename T,typename H> -std::ostream& operator<<(std::ostream& o, const CCRP<T,H>& c) { - c.Print(&o); - return o; -} - -#endif diff --git a/gi/clda/src/clda.cc b/gi/clda/src/clda.cc deleted file mode 100644 index f548997f..00000000 --- a/gi/clda/src/clda.cc +++ /dev/null @@ -1,148 +0,0 @@ -#include <iostream> -#include <vector> -#include <map> -#include <string> - -#include "timer.h" -#include "crp.h" -#include "ccrp.h" -#include "sampler.h" -#include "tdict.h" -const size_t MAX_DOC_LEN_CHARS = 10000000; - -using namespace std; - -void ShowTopWordsForTopic(const map<WordID, int>& counts) { - multimap<int, WordID> ms; - for (map<WordID,int>::const_iterator it = counts.begin(); it != counts.end(); ++it) - ms.insert(make_pair(it->second, it->first)); - int cc = 0; - for (multimap<int, WordID>::reverse_iterator it = ms.rbegin(); it != ms.rend(); ++it) { - cerr << it->first << ':' << TD::Convert(it->second) << " "; - ++cc; - if (cc==20) break; - } - cerr << endl; -} - -int main(int argc, char** argv) { - if (argc != 3) { - cerr << "Usage: " << argv[0] << " num-classes num-samples\n"; - return 1; - } - const int num_classes = atoi(argv[1]); - const int num_iterations = atoi(argv[2]); - const int burnin_size = num_iterations * 0.9; - if (num_classes < 2) { - cerr << "Must request more than 1 class\n"; - return 1; - } - if (num_iterations < 5) { - cerr << "Must request more than 5 iterations\n"; - return 1; - } - cerr << "CLASSES: " << num_classes << endl; - char* buf = new char[MAX_DOC_LEN_CHARS]; - vector<vector<int> > wji; // w[j][i] - observed word i of doc j - vector<vector<int> > zji; // z[j][i] - topic assignment for word i of doc j - cerr << "READING DOCUMENTS\n"; - while(cin) { - cin.getline(buf, MAX_DOC_LEN_CHARS); - if (buf[0] == 0) continue; - wji.push_back(vector<WordID>()); - TD::ConvertSentence(buf, &wji.back()); - } - cerr << "READ " << wji.size() << " DOCUMENTS\n"; - MT19937 rng; - cerr << "INITIALIZING RANDOM TOPIC ASSIGNMENTS\n"; - zji.resize(wji.size()); - double disc = 0.1; - double beta = 10.0; - double alpha = 50.0; - const double uniform_topic = 1.0 / num_classes; - const double uniform_word = 1.0 / TD::NumWords(); - vector<CCRP<int> > dr(zji.size(), CCRP<int>(1,1,1,1,disc, beta)); // dr[i] describes the probability of using a topic in document i - vector<CCRP<int> > wr(num_classes, CCRP<int>(1,1,1,1,disc, alpha)); // wr[k] describes the probability of generating a word in topic k - for (int j = 0; j < zji.size(); ++j) { - const size_t num_words = wji[j].size(); - vector<int>& zj = zji[j]; - const vector<int>& wj = wji[j]; - zj.resize(num_words); - for (int i = 0; i < num_words; ++i) { - int random_topic = rng.next() * num_classes; - if (random_topic == num_classes) { --random_topic; } - zj[i] = random_topic; - const int word = wj[i]; - dr[j].increment(random_topic, uniform_topic, &rng); - wr[random_topic].increment(word, uniform_word, &rng); - } - } - cerr << "SAMPLING\n"; - vector<map<WordID, int> > t2w(num_classes); - Timer timer; - SampleSet<double> ss; - ss.resize(num_classes); - double total_time = 0; - for (int iter = 0; iter < num_iterations; ++iter) { - cerr << '.'; - if (iter && iter % 10 == 0) { - total_time += timer.Elapsed(); - timer.Reset(); - double llh = 0; -#if 1 - for (int j = 0; j < dr.size(); ++j) - dr[j].resample_hyperparameters(&rng); - for (int j = 0; j < wr.size(); ++j) - wr[j].resample_hyperparameters(&rng); -#endif - - for (int j = 0; j < dr.size(); ++j) - llh += dr[j].log_crp_prob(); - for (int j = 0; j < wr.size(); ++j) - llh += wr[j].log_crp_prob(); - cerr << " [LLH=" << llh << " I=" << iter << "]\n"; - } - for (int j = 0; j < zji.size(); ++j) { - const size_t num_words = wji[j].size(); - vector<int>& zj = zji[j]; - const vector<int>& wj = wji[j]; - for (int i = 0; i < num_words; ++i) { - const int word = wj[i]; - const int cur_topic = zj[i]; - dr[j].decrement(cur_topic, &rng); - wr[cur_topic].decrement(word, &rng); - - for (int k = 0; k < num_classes; ++k) { - ss[k]= dr[j].prob(k, uniform_topic) * wr[k].prob(word, uniform_word); - } - const int new_topic = rng.SelectSample(ss); - dr[j].increment(new_topic, uniform_topic, &rng); - wr[new_topic].increment(word, uniform_word, &rng); - zj[i] = new_topic; - if (iter > burnin_size) { - ++t2w[cur_topic][word]; - } - } - } - } - for (int i = 0; i < num_classes; ++i) { - cerr << "---------------------------------\n"; - cerr << " final PYP(" << wr[i].discount() << "," << wr[i].concentration() << ")\n"; - ShowTopWordsForTopic(t2w[i]); - } - cerr << "-------------\n"; -#if 0 - for (int j = 0; j < zji.size(); ++j) { - const size_t num_words = wji[j].size(); - vector<int>& zj = zji[j]; - const vector<int>& wj = wji[j]; - zj.resize(num_words); - for (int i = 0; i < num_words; ++i) { - cerr << TD::Convert(wji[j][i]) << '(' << zj[i] << ") "; - } - cerr << endl; - } -#endif - return 0; -} - diff --git a/gi/clda/src/crp.h b/gi/clda/src/crp.h deleted file mode 100644 index 9d35857e..00000000 --- a/gi/clda/src/crp.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef _CRP_H_ -#define _CRP_H_ - -// shamelessly adapted from code by Phil Blunsom and Trevor Cohn - -#include <boost/functional/hash.hpp> -#include <tr1/unordered_map> - -#include "prob.h" - -template <typename DishType, typename Hash = boost::hash<DishType> > -class CRP { - public: - CRP(double alpha) : alpha_(alpha), palpha_(alpha), total_customers_() {} - void increment(const DishType& dish); - void decrement(const DishType& dish); - void erase(const DishType& dish) { - counts_.erase(dish); - } - inline int count(const DishType& dish) const { - const typename MapType::const_iterator i = counts_.find(dish); - if (i == counts_.end()) return 0; else return i->second; - } - inline prob_t prob(const DishType& dish, const prob_t& p0) const { - return (prob_t(count(dish)) + palpha_ * p0) / prob_t(total_customers_ + alpha_); - } - private: - typedef std::tr1::unordered_map<DishType, int, Hash> MapType; - MapType counts_; - const double alpha_; - const prob_t palpha_; - int total_customers_; -}; - -template <typename Dish, typename Hash> -void CRP<Dish,Hash>::increment(const Dish& dish) { - ++counts_[dish]; - ++total_customers_; -} - -template <typename Dish, typename Hash> -void CRP<Dish,Hash>::decrement(const Dish& dish) { - typename MapType::iterator i = counts_.find(dish); - assert(i != counts_.end()); - if (--i->second == 0) - counts_.erase(i); - --total_customers_; -} - -#endif diff --git a/gi/clda/src/slice_sampler.h b/gi/clda/src/slice_sampler.h deleted file mode 100644 index aa48a169..00000000 --- a/gi/clda/src/slice_sampler.h +++ /dev/null @@ -1,191 +0,0 @@ -//! slice-sampler.h is an MCMC slice sampler -//! -//! Mark Johnson, 1st August 2008 - -#ifndef SLICE_SAMPLER_H -#define SLICE_SAMPLER_H - -#include <algorithm> -#include <cassert> -#include <cmath> -#include <iostream> -#include <limits> - -//! slice_sampler_rfc_type{} returns the value of a user-specified -//! function if the argument is within range, or - infinity otherwise -// -template <typename F, typename Fn, typename U> -struct slice_sampler_rfc_type { - F min_x, max_x; - const Fn& f; - U max_nfeval, nfeval; - slice_sampler_rfc_type(F min_x, F max_x, const Fn& f, U max_nfeval) - : min_x(min_x), max_x(max_x), f(f), max_nfeval(max_nfeval), nfeval(0) { } - - F operator() (F x) { - if (min_x < x && x < max_x) { - assert(++nfeval <= max_nfeval); - F fx = f(x); - assert(std::isfinite(fx)); - return fx; - } - return -std::numeric_limits<F>::infinity(); - } -}; // slice_sampler_rfc_type{} - -//! slice_sampler1d() implements the univariate "range doubling" slice sampler -//! described in Neal (2003) "Slice Sampling", The Annals of Statistics 31(3), 705-767. -// -template <typename F, typename LogF, typename Uniform01> -F slice_sampler1d(const LogF& logF0, //!< log of function to sample - F x, //!< starting point - Uniform01& u01, //!< uniform [0,1) random number generator - F min_x = -std::numeric_limits<F>::infinity(), //!< minimum value of support - F max_x = std::numeric_limits<F>::infinity(), //!< maximum value of support - F w = 0.0, //!< guess at initial width - unsigned nsamples=1, //!< number of samples to draw - unsigned max_nfeval=200) //!< max number of function evaluations -{ - typedef unsigned U; - slice_sampler_rfc_type<F,LogF,U> logF(min_x, max_x, logF0, max_nfeval); - - assert(std::isfinite(x)); - - if (w <= 0.0) { // set w to a default width - if (min_x > -std::numeric_limits<F>::infinity() && max_x < std::numeric_limits<F>::infinity()) - w = (max_x - min_x)/4; - else - w = std::max(((x < 0.0) ? -x : x)/4, (F) 0.1); - } - assert(std::isfinite(w)); - - F logFx = logF(x); - for (U sample = 0; sample < nsamples; ++sample) { - F logY = logFx + log(u01()+1e-100); //! slice logFx at this value - assert(std::isfinite(logY)); - - F xl = x - w*u01(); //! lower bound on slice interval - F logFxl = logF(xl); - F xr = xl + w; //! upper bound on slice interval - F logFxr = logF(xr); - - while (logY < logFxl || logY < logFxr) // doubling procedure - if (u01() < 0.5) - logFxl = logF(xl -= xr - xl); - else - logFxr = logF(xr += xr - xl); - - F xl1 = xl; - F xr1 = xr; - while (true) { // shrinking procedure - F x1 = xl1 + u01()*(xr1 - xl1); - if (logY < logF(x1)) { - F xl2 = xl; // acceptance procedure - F xr2 = xr; - bool d = false; - while (xr2 - xl2 > 1.1*w) { - F xm = (xl2 + xr2)/2; - if ((x < xm && x1 >= xm) || (x >= xm && x1 < xm)) - d = true; - if (x1 < xm) - xr2 = xm; - else - xl2 = xm; - if (d && logY >= logF(xl2) && logY >= logF(xr2)) - goto unacceptable; - } - x = x1; - goto acceptable; - } - goto acceptable; - unacceptable: - if (x1 < x) // rest of shrinking procedure - xl1 = x1; - else - xr1 = x1; - } - acceptable: - w = (4*w + (xr1 - xl1))/5; // update width estimate - } - return x; -} - -/* -//! slice_sampler1d() implements a 1-d MCMC slice sampler. -//! It should be correct for unimodal distributions, but -//! not for multimodal ones. -// -template <typename F, typename LogP, typename Uniform01> -F slice_sampler1d(const LogP& logP, //!< log of distribution to sample - F x, //!< initial sample - Uniform01& u01, //!< uniform random number generator - F min_x = -std::numeric_limits<F>::infinity(), //!< minimum value of support - F max_x = std::numeric_limits<F>::infinity(), //!< maximum value of support - F w = 0.0, //!< guess at initial width - unsigned nsamples=1, //!< number of samples to draw - unsigned max_nfeval=200) //!< max number of function evaluations -{ - typedef unsigned U; - assert(std::isfinite(x)); - if (w <= 0.0) { - if (min_x > -std::numeric_limits<F>::infinity() && max_x < std::numeric_limits<F>::infinity()) - w = (max_x - min_x)/4; - else - w = std::max(((x < 0.0) ? -x : x)/4, 0.1); - } - // TRACE4(x, min_x, max_x, w); - F logPx = logP(x); - assert(std::isfinite(logPx)); - U nfeval = 1; - for (U sample = 0; sample < nsamples; ++sample) { - F x0 = x; - F logU = logPx + log(u01()+1e-100); - assert(std::isfinite(logU)); - F r = u01(); - F xl = std::max(min_x, x - r*w); - F xr = std::min(max_x, x + (1-r)*w); - // TRACE3(x, logPx, logU); - while (xl > min_x && logP(xl) > logU) { - xl -= w; - w *= 2; - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xl = " << xl << std::endl; - assert(nfeval < max_nfeval); - } - xl = std::max(xl, min_x); - while (xr < max_x && logP(xr) > logU) { - xr += w; - w *= 2; - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xr = " << xr << std::endl; - assert(nfeval < max_nfeval); - } - xr = std::min(xr, max_x); - while (true) { - r = u01(); - x = r*xl + (1-r)*xr; - assert(std::isfinite(x)); - logPx = logP(x); - // TRACE4(logPx, x, xl, xr); - assert(std::isfinite(logPx)); - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xl = " << xl << ", xr = " << xr << ", x = " << x << std::endl; - assert(nfeval < max_nfeval); - if (logPx > logU) - break; - else if (x > x0) - xr = x; - else - xl = x; - } - // w = (4*w + (xr-xl))/5; // gradually adjust w - } - // TRACE2(logPx, x); - return x; -} // slice_sampler1d() -*/ - -#endif // SLICE_SAMPLER_H diff --git a/gi/clda/src/timer.h b/gi/clda/src/timer.h deleted file mode 100644 index 123d9a94..00000000 --- a/gi/clda/src/timer.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef _TIMER_STATS_H_ -#define _TIMER_STATS_H_ - -#include <ctime> - -struct Timer { - Timer() { Reset(); } - void Reset() { - start_t = clock(); - } - double Elapsed() const { - const clock_t end_t = clock(); - const double elapsed = (end_t - start_t) / 1000000.0; - return elapsed; - } - private: - std::clock_t start_t; -}; - -#endif diff --git a/gi/evaluation/conditional_entropy.py b/gi/evaluation/conditional_entropy.py deleted file mode 100644 index 356d3b1d..00000000 --- a/gi/evaluation/conditional_entropy.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python - -import sys, math, itertools, getopt - -def usage(): - print >>sys.stderr, 'Usage:', sys.argv[0], '[-s slash_threshold] input-1 input-2' - sys.exit(0) - -optlist, args = getopt.getopt(sys.argv[1:], 'hs:') -slash_threshold = None -for opt, arg in optlist: - if opt == '-s': - slash_threshold = int(arg) - else: - usage() -if len(args) != 2: - usage() - -ginfile = open(args[0]) -pinfile = open(args[1]) - -# evaluating: H(G | P) = sum_{g,p} p(g,p) log { p(p) / p(g,p) } -# = sum_{g,p} c(g,p)/N { log c(p) - log N - log c(g,p) + log N } -# = 1/N sum_{g,p} c(g,p) { log c(p) - log c(g,p) } -# where G = gold, P = predicted, N = number of events - -N = 0 -gold_frequencies = {} -predict_frequencies = {} -joint_frequencies = {} - -for gline, pline in itertools.izip(ginfile, pinfile): - gparts = gline.split('||| ')[1].split() - pparts = pline.split('||| ')[1].split() - assert len(gparts) == len(pparts) - - for gpart, ppart in zip(gparts, pparts): - gtag = gpart.split(':',1)[1] - ptag = ppart.split(':',1)[1] - - if slash_threshold == None or gtag.count('/') + gtag.count('\\') <= slash_threshold: - joint_frequencies.setdefault((gtag, ptag), 0) - joint_frequencies[gtag,ptag] += 1 - - predict_frequencies.setdefault(ptag, 0) - predict_frequencies[ptag] += 1 - - gold_frequencies.setdefault(gtag, 0) - gold_frequencies[gtag] += 1 - - N += 1 - -hg2p = 0 -hp2g = 0 -for (gtag, ptag), cgp in joint_frequencies.items(): - hp2g += cgp * (math.log(predict_frequencies[ptag], 2) - math.log(cgp, 2)) - hg2p += cgp * (math.log(gold_frequencies[gtag], 2) - math.log(cgp, 2)) -hg2p /= N -hp2g /= N - -print 'H(P|G)', hg2p, 'H(G|P)', hp2g, 'VI', hg2p + hp2g diff --git a/gi/evaluation/confusion_matrix.py b/gi/evaluation/confusion_matrix.py deleted file mode 100644 index 2dd7aa47..00000000 --- a/gi/evaluation/confusion_matrix.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python - -import sys, math, itertools, getopt - -def usage(): - print >>sys.stderr, 'Usage:', sys.argv[0], '[-s slash_threshold] [-p output] [-m] input-1 input-2' - sys.exit(0) - -optlist, args = getopt.getopt(sys.argv[1:], 'hs:mp:') -slash_threshold = None -output_fname = None -show_matrix = False -for opt, arg in optlist: - if opt == '-s': - slash_threshold = int(arg) - elif opt == '-p': - output_fname = arg - elif opt == '-m': - show_matrix = True - else: - usage() -if len(args) != 2 or (not show_matrix and not output_fname): - usage() - -ginfile = open(args[0]) -pinfile = open(args[1]) - -if output_fname: - try: - import Image, ImageDraw - except ImportError: - print >>sys.stderr, "Error: Python Image Library not available. Did you forget to set your PYTHONPATH environment variable?" - sys.exit(1) - -N = 0 -gold_frequencies = {} -predict_frequencies = {} -joint_frequencies = {} - -for gline, pline in itertools.izip(ginfile, pinfile): - gparts = gline.split('||| ')[1].split() - pparts = pline.split('||| ')[1].split() - assert len(gparts) == len(pparts) - - for gpart, ppart in zip(gparts, pparts): - gtag = gpart.split(':',1)[1] - ptag = ppart.split(':',1)[1] - - if slash_threshold == None or gtag.count('/') + gtag.count('\\') <= slash_threshold: - joint_frequencies.setdefault((gtag, ptag), 0) - joint_frequencies[gtag,ptag] += 1 - - predict_frequencies.setdefault(ptag, 0) - predict_frequencies[ptag] += 1 - - gold_frequencies.setdefault(gtag, 0) - gold_frequencies[gtag] += 1 - - N += 1 - -# find top tags -gtags = gold_frequencies.items() -gtags.sort(lambda x,y: x[1]-y[1]) -gtags.reverse() -#gtags = gtags[:50] - -preds = predict_frequencies.items() -preds.sort(lambda x,y: x[1]-y[1]) -preds.reverse() - -if show_matrix: - print '%7s %7s' % ('pred', 'cnt'), - for gtag, gcount in gtags: print '%7s' % gtag, - print - print '=' * 80 - - for ptag, pcount in preds: - print '%7s %7d' % (ptag, pcount), - for gtag, gcount in gtags: - print '%7d' % joint_frequencies.get((gtag, ptag), 0), - print - - print '%7s %7d' % ('total', N), - for gtag, gcount in gtags: print '%7d' % gcount, - print - -if output_fname: - offset=10 - - image = Image.new("RGB", (len(preds), len(gtags)), (255, 255, 255)) - #hsl(hue, saturation%, lightness%) - - # re-sort preds to get a better diagonal - ptags=[] - if True: - ptags = map(lambda (p,c): p, preds) - else: - remaining = set(predict_frequencies.keys()) - for y, (gtag, gcount) in enumerate(gtags): - best = (None, 0) - for ptag in remaining: - #pcount = predict_frequencies[ptag] - p = joint_frequencies.get((gtag, ptag), 0)# / float(pcount) - if p > best[1]: best = (ptag, p) - ptags.append(ptag) - remaining.remove(ptag) - if not remaining: break - - print 'Predicted tag ordering:', ' '.join(ptags) - print 'Gold tag ordering:', ' '.join(map(lambda (t,c): t, gtags)) - - draw = ImageDraw.Draw(image) - for x, ptag in enumerate(ptags): - pcount = predict_frequencies[ptag] - minval = math.log(offset) - maxval = math.log(pcount + offset) - for y, (gtag, gcount) in enumerate(gtags): - f = math.log(offset + joint_frequencies.get((gtag, ptag), 0)) - z = int(240. * (maxval - f) / float(maxval - minval)) - #print x, y, z, f, maxval - draw.point([(x,y)], fill='hsl(%d, 100%%, 50%%)' % z) - del draw - image.save(output_fname) diff --git a/gi/evaluation/entropy.py b/gi/evaluation/entropy.py deleted file mode 100644 index ec1ef502..00000000 --- a/gi/evaluation/entropy.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -import sys, math, itertools, getopt - -def usage(): - print >>sys.stderr, 'Usage:', sys.argv[0], '[-s slash_threshold] input file' - sys.exit(0) - -optlist, args = getopt.getopt(sys.argv[1:], 'hs:') -slash_threshold = None -for opt, arg in optlist: - if opt == '-s': - slash_threshold = int(arg) - else: - usage() -if len(args) != 1: - usage() - -infile = open(args[0]) -N = 0 -frequencies = {} - -for line in infile: - - for part in line.split('||| ')[1].split(): - tag = part.split(':',1)[1] - - if slash_threshold == None or tag.count('/') + tag.count('\\') <= slash_threshold: - frequencies.setdefault(tag, 0) - frequencies[tag] += 1 - N += 1 - -h = 0 -for tag, c in frequencies.items(): - h -= c * (math.log(c, 2) - math.log(N, 2)) -h /= N - -print 'entropy', h diff --git a/gi/evaluation/extract_ccg_labels.py b/gi/evaluation/extract_ccg_labels.py deleted file mode 100644 index e0034648..00000000 --- a/gi/evaluation/extract_ccg_labels.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python - -# -# Takes spans input along with treebank and spits out CG style categories for each span. -# spans = output from CDEC's extools/extractor with --base_phrase_spans option -# treebank = PTB format, one tree per line -# -# Output is in CDEC labelled-span format -# - -import sys, itertools, tree - -tinfile = open(sys.argv[1]) -einfile = open(sys.argv[2]) - -def number_leaves(node, next=0): - left, right = None, None - for child in node.children: - l, r = number_leaves(child, next) - next = max(next, r+1) - if left == None or l < left: - left = l - if right == None or r > right: - right = r - - #print node, left, right, next - if left == None or right == None: - assert not node.children - left = right = next - - node.left = left - node.right = right - - return left, right - -def ancestor(node, indices): - #print node, node.left, node.right, indices - # returns the deepest node covering all the indices - if min(indices) >= node.left and max(indices) <= node.right: - # try the children - for child in node.children: - x = ancestor(child, indices) - if x: return x - return node - else: - return None - -def frontier(node, indices): - #print 'frontier for node', node, 'indices', indices - if node.left > max(indices) or node.right < min(indices): - #print '\toutside' - return [node] - elif node.children: - #print '\tcovering at least part' - ns = [] - for child in node.children: - n = frontier(child, indices) - ns.extend(n) - return ns - else: - return [node] - -def project_heads(node): - #print 'project_heads', node - is_head = node.data.tag.endswith('-HEAD') - if node.children: - found = 0 - for child in node.children: - x = project_heads(child) - if x: - node.data.tag = x - found += 1 - assert found == 1 - elif is_head: - node.data.tag = node.data.tag[:-len('-HEAD')] - - if is_head: - return node.data.tag - else: - return None - -for tline, eline in itertools.izip(tinfile, einfile): - if tline.strip() != '(())': - if tline.startswith('( '): - tline = tline[2:-1].strip() - tr = tree.parse_PST(tline) - if tr != None: - number_leaves(tr) - #project_heads(tr) # assumes Bikel-style head annotation for the input trees - else: - tr = None - - parts = eline.strip().split(" ||| ") - zh, en = parts[:2] - spans = parts[-1] - print '|||', - for span in spans.split(): - sps = span.split(":") - i, j, x, y = map(int, sps[0].split("-")) - - if tr: - a = ancestor(tr, range(x,y)) - try: - fs = frontier(a, range(x,y)) - except: - print >>sys.stderr, "problem with line", tline.strip(), "--", eline.strip() - raise - - #print x, y - #print 'ancestor', a - #print 'frontier', fs - - cat = a.data.tag - for f in fs: - if f.right < x: - cat += '\\' + f.data.tag - else: - break - fs.reverse() - for f in fs: - if f.left >= y: - cat += '/' + f.data.tag - else: - break - else: - cat = 'FAIL' - - print '%d-%d:%s' % (x, y, cat), - print diff --git a/gi/evaluation/tree.py b/gi/evaluation/tree.py deleted file mode 100644 index 702d80b6..00000000 --- a/gi/evaluation/tree.py +++ /dev/null @@ -1,485 +0,0 @@ -import re, sys - -class Symbol: - def __init__(self, nonterm, term=None, var=None): - assert not (term != None and var != None) - self.tag = nonterm - self.token = term - self.variable = var - - def is_variable(self): - return self.variable != None - - def __eq__(self, other): - return self.tag == other.tag and self.token == other.token and self.variable == other.variable - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return hash((self.tag, self.token, self.variable)) - - def __repr__(self): - return str(self) - - def __cmp__(self, other): - return cmp((self.tag, self.token, self.variable), - (other.tag, other.token, other.variable)) - - def __str__(self): - parts = [] - if False: # DEPENDENCY - if self.token: - parts.append(str(self.token)) - elif self.variable != None: - parts.append('#%d' % self.variable) - if self.tag: - parts.append(str(self.tag)) - return '/'.join(parts) - else: - if self.tag: - parts.append(str(self.tag)) - if self.token: - parts.append(str(self.token)) - elif self.variable != None: - parts.append('#%d' % self.variable) - return ' '.join(parts) - -class TreeNode: - def __init__(self, data, children=None, order=-1): - self.data = data - self.children = [] - self.order = order - self.parent = None - if children: self.children = children - - def insert(self, child): - self.children.append(child) - child.parent = self - - def leaves(self): - ls = [] - for node in self.xtraversal(): - if not node.children: - ls.append(node.data) - return ls - - def leaf_nodes(self): - ls = [] - for node in self.xtraversal(): - if not node.children: - ls.append(node) - return ls - - def max_depth(self): - d = 1 - for child in self.children: - d = max(d, 1 + child.max_depth()) - if not self.children and self.data.token: - d = 2 - return d - - def max_width(self): - w = 0 - for child in self.children: - w += child.max_width() - return max(1, w) - - def num_internal_nodes(self): - if self.children: - n = 1 - for child in self.children: - n += child.num_internal_nodes() - return n - elif self.data.token: - return 1 - else: - return 0 - - def postorder_traversal(self, visit): - """ - Postorder traversal; no guarantee that terminals will be read in the - correct order for dep. trees. - """ - for child in self.children: - child.traversal(visit) - visit(self) - - def traversal(self, visit): - """ - Preorder for phrase structure trees, and inorder for dependency trees. - In both cases the terminals will be read off in the correct order. - """ - visited_self = False - if self.order <= 0: - visited_self = True - visit(self) - - for i, child in enumerate(self.children): - child.traversal(visit) - if i + 1 == self.order: - visited_self = True - visit(self) - - assert visited_self - - def xpostorder_traversal(self): - for child in self.children: - for node in child.xpostorder_traversal(): - yield node - yield self - - def xtraversal(self): - visited_self = False - if self.order <= 0: - visited_self = True - yield self - - for i, child in enumerate(self.children): - for d in child.xtraversal(): - yield d - - if i + 1 == self.order: - visited_self = True - yield self - - assert visited_self - - def xpostorder_traversal(self): - for i, child in enumerate(self.children): - for d in child.xpostorder_traversal(): - yield d - yield self - - def edges(self): - es = [] - self.traverse_edges(lambda h,c: es.append((h,c))) - return es - - def traverse_edges(self, visit): - for child in self.children: - visit(self.data, child.data) - child.traverse_edges(visit) - - def subtrees(self, include_self=False): - st = [] - if include_self: - stack = [self] - else: - stack = self.children[:] - - while stack: - node = stack.pop() - st.append(node) - stack.extend(node.children) - return st - - def find_parent(self, node): - try: - index = self.children.index(node) - return self, index - except ValueError: - for child in self.children: - if isinstance(child, TreeNode): - r = child.find_parent(node) - if r: return r - return None - - def is_ancestor_of(self, node): - if self == node: - return True - for child in self.children: - if child.is_ancestor_of(child): - return True - return False - - def find(self, node): - if self == node: - return self - for child in self.children: - if isinstance(child, TreeNode): - r = child.find(node) - if r: return r - else: - if child == node: - return r - return None - - def equals_ignorecase(self, other): - if not isinstance(other, TreeNode): - return False - if self.data != other.data: - return False - if len(self.children) != len(other.children): - return False - for mc, oc in zip(self.children, other.children): - if isinstance(mc, TreeNode): - if not mc.equals_ignorecase(oc): - return False - else: - if mc.lower() != oc.lower(): - return False - return True - - def node_number(self, numbering, next=0): - if self.order <= 0: - numbering[id(self)] = next - next += 1 - - for i, child in enumerate(self.children): - next = child.node_number(numbering, next) - if i + 1 == self.order: - numbering[id(self)] = next - next += 1 - - return next - - def display_conll(self, out): - numbering = {} - self.node_number(numbering) - next = 0 - self.children[0].traversal(lambda x: \ - out.write('%d\t%s\t%s\t%s\t%s\t_\t%d\tLAB\n' \ - % (numbering[id(x)], x.data.token, x.data.token, - x.data.tag, x.data.tag, numbering[id(x.parent)]))) - out.write('\n') - - def size(self): - sz = 1 - for child in self.children: - sz += child.size() - return sz - - def __eq__(self, other): - if isinstance(other, TreeNode) and self.data == other.data \ - and self.children == other.children: - return True - return False - - def __cmp__(self, other): - if not isinstance(other, TreeNode): return 1 - n = cmp(self.data, other.data) - if n != 0: return n - n = len(self.children) - len(other.children) - if n != 0: return n - for sc, oc in zip(self.children, other.children): - n = cmp(sc, oc) - if n != 0: return n - return 0 - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash((self.data, tuple(self.children))) - - def __repr__(self): - return str(self) - - def __str__(self): - s = '(' - space = False - if self.order <= 0: - s += str(self.data) - space = True - for i, child in enumerate(self.children): - if space: s += ' ' - s += str(child) - space = True - if i+1 == self.order: - s += ' ' + str(self.data) - return s + ')' - -def read_PSTs(fname): - infile = open(fname) - trees = [] - for line in infile: - trees.append(parse_PST(line.strip())) - infile.close() - return trees - -def parse_PST_multiline(infile, hash_is_var=True): - buf = '' - num_open = 0 - while True: - line = infile.readline() - if not line: - return None - buf += ' ' + line.rstrip() - num_open += line.count('(') - line.count(')') - if num_open == 0: - break - - return parse_PST(buf, hash_is_var) - -def parse_PST(line, hash_is_var=True): - line = line.rstrip() - if not line or line.lower() == 'null': - return None - - # allow either (a/DT) or (DT a) - #parts_re = re.compile(r'(\(*)([^/)]*)(?:/([^)]*))?(\)*)$') - - # only allow (DT a) - parts_re = re.compile(r'(\(*)([^)]*)(\)*)$') - - root = TreeNode(Symbol('TOP')) - stack = [root] - for part in line.rstrip().split(): - m = parts_re.match(part) - #opening, tok_or_tag, tag, closing = m.groups() - opening, tok_or_tag, closing = m.groups() - tag = None - #print 'token', part, 'bits', m.groups() - for i in opening: - node = TreeNode(Symbol(None)) - stack[-1].insert(node) - stack.append(node) - - if tag: - stack[-1].data.tag = tag - if hash_is_var and tok_or_tag.startswith('#'): - stack[-1].data.variable = int(tok_or_tag[1:]) - else: - stack[-1].data.token = tok_or_tag - else: - if stack[-1].data.tag == None: - stack[-1].data.tag = tok_or_tag - else: - if hash_is_var and tok_or_tag.startswith('#'): - try: - stack[-1].data.variable = int(tok_or_tag[1:]) - except ValueError: # it's really a token! - #print >>sys.stderr, 'Warning: # used for token:', tok_or_tag - stack[-1].data.token = tok_or_tag - else: - stack[-1].data.token = tok_or_tag - - for i in closing: - stack.pop() - - #assert str(root.children[0]) == line - return root.children[0] - -def read_DTs(fname): - infile = open(fname) - trees = [] - while True: - t = parse_DT(infile) - if t: trees.append(t) - else: break - infile.close() - return trees - -def read_bracketed_DTs(fname): - infile = open(fname) - trees = [] - for line in infile: - trees.append(parse_bracketed_DT(line)) - infile.close() - return trees - -def parse_DT(infile): - tokens = [Symbol('ROOT')] - children = {} - - for line in infile: - parts = line.rstrip().split() - #print parts - if not parts: break - index = len(tokens) - token = parts[1] - tag = parts[3] - parent = int(parts[6]) - if token.startswith('#'): - tokens.append(Symbol(tag, var=int(token[1:]))) - else: - tokens.append(Symbol(tag, token)) - children.setdefault(parent, set()).add(index) - - if len(tokens) == 1: return None - - root = TreeNode(Symbol('ROOT'), [], 0) - schedule = [] - for child in sorted(children[0]): - schedule.append((root, child)) - - while schedule: - parent, index = schedule[0] - del schedule[0] - - node = TreeNode(tokens[index]) - node.order = 0 - parent.insert(node) - - for child in sorted(children.get(index, [])): - schedule.append((node, child)) - if child < index: - node.order += 1 - - return root - -_bracket_split_re = re.compile(r'([(]*)([^)/]*)(?:/([^)]*))?([)]*)') - -def parse_bracketed_DT(line, insert_root=True): - line = line.rstrip() - if not line or line == 'NULL': return None - #print line - - root = TreeNode(Symbol('ROOT')) - stack = [root] - for part in line.rstrip().split(): - m = _bracket_split_re.match(part) - - for c in m.group(1): - node = TreeNode(Symbol(None)) - stack[-1].insert(node) - stack.append(node) - - if m.group(3) != None: - if m.group(2).startswith('#'): - stack[-1].data.variable = int(m.group(2)[1:]) - else: - stack[-1].data.token = m.group(2) - stack[-1].data.tag = m.group(3) - else: - stack[-1].data.tag = m.group(2) - stack[-1].order = len(stack[-1].children) - # FIXME: also check for vars - - for c in m.group(4): - stack.pop() - - assert len(stack) == 1 - if not insert_root or root.children[0].data.tag == 'ROOT': - return root.children[0] - else: - return root - -_bracket_split_notag_re = re.compile(r'([(]*)([^)/]*)([)]*)') - -def parse_bracketed_untagged_DT(line): - line = line.rstrip() - if not line or line == 'NULL': return None - - root = TreeNode(Symbol('TOP')) - stack = [root] - for part in line.rstrip().split(): - m = _bracket_split_notag_re.match(part) - - for c in m.group(1): - node = TreeNode(Symbol(None)) - stack[-1].insert(node) - stack.append(node) - - if stack[-1].data.token == None: - stack[-1].data.token = m.group(2) - stack[-1].order = len(stack[-1].children) - else: - child = TreeNode(Symbol(nonterm=None, term=m.group(2))) - stack[-1].insert(child) - - for c in m.group(3): - stack.pop() - - return root.children[0] diff --git a/gi/markov_al/Makefile.am b/gi/markov_al/Makefile.am deleted file mode 100644 index fe3e3349..00000000 --- a/gi/markov_al/Makefile.am +++ /dev/null @@ -1,6 +0,0 @@ -bin_PROGRAMS = ml - -ml_SOURCES = ml.cc - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -funroll-loops -I$(top_srcdir)/utils $(GTEST_CPPFLAGS) -I$(top_srcdir)/decoder -AM_LDFLAGS = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/utils/libutils.a -lz diff --git a/gi/markov_al/README b/gi/markov_al/README deleted file mode 100644 index 9c10f7cd..00000000 --- a/gi/markov_al/README +++ /dev/null @@ -1,2 +0,0 @@ -Experimental translation models with Markovian dependencies. - diff --git a/gi/markov_al/ml.cc b/gi/markov_al/ml.cc deleted file mode 100644 index 1e71edd6..00000000 --- a/gi/markov_al/ml.cc +++ /dev/null @@ -1,470 +0,0 @@ -#include <iostream> -#include <tr1/unordered_map> - -#include <boost/shared_ptr.hpp> -#include <boost/functional.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "tdict.h" -#include "filelib.h" -#include "sampler.h" -#include "ccrp_onetable.h" -#include "array2d.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -void PrintTopCustomers(const CCRP_OneTable<WordID>& crp) { - for (CCRP_OneTable<WordID>::const_iterator it = crp.begin(); it != crp.end(); ++it) { - cerr << " " << TD::Convert(it->first) << " = " << it->second << endl; - } -} - -void PrintAlignment(const vector<WordID>& src, const vector<WordID>& trg, const vector<unsigned char>& a) { - cerr << TD::GetString(src) << endl << TD::GetString(trg) << endl; - Array2D<bool> al(src.size(), trg.size()); - for (int i = 0; i < a.size(); ++i) - if (a[i] != 255) al(a[i], i) = true; - cerr << al << endl; -} - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("input,i",po::value<string>(),"Read parallel data from") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -struct Unigram; -struct Bigram { - Bigram() : trg(), cond() {} - Bigram(WordID prev, WordID cur, WordID t) : trg(t) { cond.first = prev; cond.second = cur; } - const pair<WordID,WordID>& ConditioningPair() const { - return cond; - } - WordID& prev_src() { return cond.first; } - WordID& cur_src() { return cond.second; } - const WordID& prev_src() const { return cond.first; } - const WordID& cur_src() const { return cond.second; } - WordID trg; - private: - pair<WordID, WordID> cond; -}; - -struct Unigram { - Unigram() : cur_src(), trg() {} - Unigram(WordID s, WordID t) : cur_src(s), trg(t) {} - WordID cur_src; - WordID trg; -}; - -ostream& operator<<(ostream& os, const Bigram& b) { - os << "( " << TD::Convert(b.trg) << " | " << TD::Convert(b.prev_src()) << " , " << TD::Convert(b.cur_src()) << " )"; - return os; -} - -ostream& operator<<(ostream& os, const Unigram& u) { - os << "( " << TD::Convert(u.trg) << " | " << TD::Convert(u.cur_src) << " )"; - return os; -} - -bool operator==(const Bigram& a, const Bigram& b) { - return a.trg == b.trg && a.cur_src() == b.cur_src() && a.prev_src() == b.prev_src(); -} - -bool operator==(const Unigram& a, const Unigram& b) { - return a.trg == b.trg && a.cur_src == b.cur_src; -} - -size_t hash_value(const Bigram& b) { - size_t h = boost::hash_value(b.prev_src()); - boost::hash_combine(h, boost::hash_value(b.cur_src())); - boost::hash_combine(h, boost::hash_value(b.trg)); - return h; -} - -size_t hash_value(const Unigram& u) { - size_t h = boost::hash_value(u.cur_src); - boost::hash_combine(h, boost::hash_value(u.trg)); - return h; -} - -void ReadParallelCorpus(const string& filename, - vector<vector<WordID> >* f, - vector<vector<WordID> >* e, - set<WordID>* vocab_f, - set<WordID>* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector<WordID> tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector<int>()); - f->push_back(vector<int>()); - vector<int>& le = e->back(); - vector<int>& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -struct UnigramModel { - UnigramModel(size_t src_voc_size, size_t trg_voc_size) : - unigrams(TD::NumWords() + 1, CCRP_OneTable<WordID>(1,1,1,1)), - p0(1.0 / trg_voc_size) {} - - void increment(const Bigram& b) { - unigrams[b.cur_src()].increment(b.trg); - } - - void decrement(const Bigram& b) { - unigrams[b.cur_src()].decrement(b.trg); - } - - double prob(const Bigram& b) const { - const double q0 = unigrams[b.cur_src()].prob(b.trg, p0); - return q0; - } - - double LogLikelihood() const { - double llh = 0; - for (unsigned i = 0; i < unigrams.size(); ++i) { - const CCRP_OneTable<WordID>& crp = unigrams[i]; - if (crp.num_customers() > 0) { - llh += crp.log_crp_prob(); - llh += crp.num_tables() * log(p0); - } - } - return llh; - } - - void ResampleHyperparameters(MT19937* rng) { - for (unsigned i = 0; i < unigrams.size(); ++i) - unigrams[i].resample_hyperparameters(rng); - } - - vector<CCRP_OneTable<WordID> > unigrams; // unigrams[src].prob(trg, p0) = p(trg|src) - - const double p0; -}; - -struct BigramModel { - BigramModel(size_t src_voc_size, size_t trg_voc_size) : - unigrams(TD::NumWords() + 1, CCRP_OneTable<WordID>(1,1,1,1)), - p0(1.0 / trg_voc_size) {} - - void increment(const Bigram& b) { - BigramMap::iterator it = bigrams.find(b.ConditioningPair()); - if (it == bigrams.end()) { - it = bigrams.insert(make_pair(b.ConditioningPair(), CCRP_OneTable<WordID>(1,1,1,1))).first; - } - if (it->second.increment(b.trg)) - unigrams[b.cur_src()].increment(b.trg); - } - - void decrement(const Bigram& b) { - BigramMap::iterator it = bigrams.find(b.ConditioningPair()); - assert(it != bigrams.end()); - if (it->second.decrement(b.trg)) { - unigrams[b.cur_src()].decrement(b.trg); - if (it->second.num_customers() == 0) - bigrams.erase(it); - } - } - - double prob(const Bigram& b) const { - const double q0 = unigrams[b.cur_src()].prob(b.trg, p0); - const BigramMap::const_iterator it = bigrams.find(b.ConditioningPair()); - if (it == bigrams.end()) return q0; - return it->second.prob(b.trg, q0); - } - - double LogLikelihood() const { - double llh = 0; - for (unsigned i = 0; i < unigrams.size(); ++i) { - const CCRP_OneTable<WordID>& crp = unigrams[i]; - if (crp.num_customers() > 0) { - llh += crp.log_crp_prob(); - llh += crp.num_tables() * log(p0); - } - } - for (BigramMap::const_iterator it = bigrams.begin(); it != bigrams.end(); ++it) { - const CCRP_OneTable<WordID>& crp = it->second; - const WordID cur_src = it->first.second; - llh += crp.log_crp_prob(); - for (CCRP_OneTable<WordID>::const_iterator bit = crp.begin(); bit != crp.end(); ++bit) { - llh += log(unigrams[cur_src].prob(bit->second, p0)); - } - } - return llh; - } - - void ResampleHyperparameters(MT19937* rng) { - for (unsigned i = 0; i < unigrams.size(); ++i) - unigrams[i].resample_hyperparameters(rng); - for (BigramMap::iterator it = bigrams.begin(); it != bigrams.end(); ++it) - it->second.resample_hyperparameters(rng); - } - - typedef unordered_map<pair<WordID,WordID>, CCRP_OneTable<WordID>, boost::hash<pair<WordID,WordID> > > BigramMap; - BigramMap bigrams; // bigrams[(src-1,src)].prob(trg, q0) = p(trg|src,src-1) - vector<CCRP_OneTable<WordID> > unigrams; // unigrams[src].prob(trg, p0) = p(trg|src) - - const double p0; -}; - -struct BigramAlignmentModel { - BigramAlignmentModel(size_t src_voc_size, size_t trg_voc_size) : bigrams(TD::NumWords() + 1, CCRP_OneTable<WordID>(1,1,1,1)), p0(1.0 / src_voc_size) {} - void increment(WordID prev, WordID next) { - bigrams[prev].increment(next); // hierarchy? - } - void decrement(WordID prev, WordID next) { - bigrams[prev].decrement(next); // hierarchy? - } - double prob(WordID prev, WordID next) { - return bigrams[prev].prob(next, p0); - } - double LogLikelihood() const { - double llh = 0; - for (unsigned i = 0; i < bigrams.size(); ++i) { - const CCRP_OneTable<WordID>& crp = bigrams[i]; - if (crp.num_customers() > 0) { - llh += crp.log_crp_prob(); - llh += crp.num_tables() * log(p0); - } - } - return llh; - } - - vector<CCRP_OneTable<WordID> > bigrams; // bigrams[prev].prob(next, p0) = p(next|prev) - const double p0; -}; - -struct Alignment { - vector<unsigned char> a; -}; - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned samples = conf["samples"].as<unsigned>(); - - boost::shared_ptr<MT19937> prng; - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector<vector<WordID> > corpuse, corpusf; - set<WordID> vocabe, vocabf; - cerr << "Reading corpus...\n"; - ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - const size_t corpus_len = corpusf.size(); - const WordID kNULL = TD::Convert("<eps>"); - const WordID kBOS = TD::Convert("<s>"); - const WordID kEOS = TD::Convert("</s>"); - Bigram TT(kBOS, TD::Convert("我"), TD::Convert("i")); - Bigram TT2(kBOS, TD::Convert("要"), TD::Convert("i")); - - UnigramModel model(vocabf.size(), vocabe.size()); - vector<Alignment> alignments(corpus_len); - for (unsigned ci = 0; ci < corpus_len; ++ci) { - const vector<WordID>& src = corpusf[ci]; - const vector<WordID>& trg = corpuse[ci]; - vector<unsigned char>& alg = alignments[ci].a; - alg.resize(trg.size()); - int lenp1 = src.size() + 1; - WordID prev_src = kBOS; - for (int j = 0; j < trg.size(); ++j) { - int samp = lenp1 * rng.next(); - --samp; - if (samp < 0) samp = 255; - alg[j] = samp; - WordID cur_src = (samp == 255 ? kNULL : src[alg[j]]); - Bigram b(prev_src, cur_src, trg[j]); - model.increment(b); - prev_src = cur_src; - } - Bigram b(prev_src, kEOS, kEOS); - model.increment(b); - } - cerr << "Initial LLH: " << model.LogLikelihood() << endl; - - SampleSet<double> ss; - for (unsigned si = 0; si < 50; ++si) { - for (unsigned ci = 0; ci < corpus_len; ++ci) { - const vector<WordID>& src = corpusf[ci]; - const vector<WordID>& trg = corpuse[ci]; - vector<unsigned char>& alg = alignments[ci].a; - WordID prev_src = kBOS; - for (unsigned j = 0; j < trg.size(); ++j) { - unsigned char& a_j = alg[j]; - WordID cur_e_a_j = (a_j == 255 ? kNULL : src[a_j]); - Bigram b(prev_src, cur_e_a_j, trg[j]); - //cerr << "DEC: " << b << "\t" << nextb << endl; - model.decrement(b); - ss.clear(); - for (unsigned i = 0; i <= src.size(); ++i) { - const WordID cur_src = (i ? src[i-1] : kNULL); - b.cur_src() = cur_src; - ss.add(model.prob(b)); - } - int sampled_a_j = rng.SelectSample(ss); - a_j = (sampled_a_j ? sampled_a_j - 1 : 255); - cur_e_a_j = (a_j == 255 ? kNULL : src[a_j]); - b.cur_src() = cur_e_a_j; - //cerr << "INC: " << b << "\t" << nextb << endl; - model.increment(b); - prev_src = cur_e_a_j; - } - } - cerr << '.' << flush; - if (si % 10 == 9) { - cerr << "[LLH prev=" << model.LogLikelihood(); - //model.ResampleHyperparameters(&rng); - cerr << " new=" << model.LogLikelihood() << "]\n"; - //pair<WordID,WordID> xx = make_pair(kBOS, TD::Convert("我")); - //PrintTopCustomers(model.bigrams.find(xx)->second); - cerr << "p(" << TT << ") = " << model.prob(TT) << endl; - cerr << "p(" << TT2 << ") = " << model.prob(TT2) << endl; - PrintAlignment(corpusf[0], corpuse[0], alignments[0].a); - } - } - { - // MODEL 2 - BigramModel model(vocabf.size(), vocabe.size()); - BigramAlignmentModel amodel(vocabf.size(), vocabe.size()); - for (unsigned ci = 0; ci < corpus_len; ++ci) { - const vector<WordID>& src = corpusf[ci]; - const vector<WordID>& trg = corpuse[ci]; - vector<unsigned char>& alg = alignments[ci].a; - WordID prev_src = kBOS; - for (int j = 0; j < trg.size(); ++j) { - WordID cur_src = (alg[j] == 255 ? kNULL : src[alg[j]]); - Bigram b(prev_src, cur_src, trg[j]); - model.increment(b); - amodel.increment(prev_src, cur_src); - prev_src = cur_src; - } - amodel.increment(prev_src, kEOS); - Bigram b(prev_src, kEOS, kEOS); - model.increment(b); - } - cerr << "Initial LLH: " << model.LogLikelihood() << " " << amodel.LogLikelihood() << endl; - - SampleSet<double> ss; - for (unsigned si = 0; si < samples; ++si) { - for (unsigned ci = 0; ci < corpus_len; ++ci) { - const vector<WordID>& src = corpusf[ci]; - const vector<WordID>& trg = corpuse[ci]; - vector<unsigned char>& alg = alignments[ci].a; - WordID prev_src = kBOS; - for (unsigned j = 0; j < trg.size(); ++j) { - unsigned char& a_j = alg[j]; - WordID cur_e_a_j = (a_j == 255 ? kNULL : src[a_j]); - Bigram b(prev_src, cur_e_a_j, trg[j]); - WordID next_src = kEOS; - WordID next_trg = kEOS; - if (j < (trg.size() - 1)) { - next_src = (alg[j+1] == 255 ? kNULL : src[alg[j + 1]]); - next_trg = trg[j + 1]; - } - Bigram nextb(cur_e_a_j, next_src, next_trg); - //cerr << "DEC: " << b << "\t" << nextb << endl; - model.decrement(b); - model.decrement(nextb); - amodel.decrement(prev_src, cur_e_a_j); - amodel.decrement(cur_e_a_j, next_src); - ss.clear(); - for (unsigned i = 0; i <= src.size(); ++i) { - const WordID cur_src = (i ? src[i-1] : kNULL); - b.cur_src() = cur_src; - ss.add(model.prob(b) * model.prob(nextb) * amodel.prob(prev_src, cur_src) * amodel.prob(cur_src, next_src)); - //cerr << log(ss[ss.size() - 1]) << "\t" << b << endl; - } - int sampled_a_j = rng.SelectSample(ss); - a_j = (sampled_a_j ? sampled_a_j - 1 : 255); - cur_e_a_j = (a_j == 255 ? kNULL : src[a_j]); - b.cur_src() = cur_e_a_j; - nextb.prev_src() = cur_e_a_j; - //cerr << "INC: " << b << "\t" << nextb << endl; - //exit(1); - model.increment(b); - model.increment(nextb); - amodel.increment(prev_src, cur_e_a_j); - amodel.increment(cur_e_a_j, next_src); - prev_src = cur_e_a_j; - } - } - cerr << '.' << flush; - if (si % 10 == 9) { - cerr << "[LLH prev=" << (model.LogLikelihood() + amodel.LogLikelihood()); - //model.ResampleHyperparameters(&rng); - cerr << " new=" << model.LogLikelihood() << "]\n"; - pair<WordID,WordID> xx = make_pair(kBOS, TD::Convert("我")); - cerr << "p(" << TT << ") = " << model.prob(TT) << endl; - cerr << "p(" << TT2 << ") = " << model.prob(TT2) << endl; - pair<WordID,WordID> xx2 = make_pair(kBOS, TD::Convert("要")); - PrintTopCustomers(model.bigrams.find(xx)->second); - //PrintTopCustomers(amodel.bigrams[TD::Convert("<s>")]); - //PrintTopCustomers(model.unigrams[TD::Convert("<eps>")]); - PrintAlignment(corpusf[0], corpuse[0], alignments[0].a); - } - } - } - return 0; -} - diff --git a/gi/morf-segmentation/filter_docs.pl b/gi/morf-segmentation/filter_docs.pl deleted file mode 100755 index a78575da..00000000 --- a/gi/morf-segmentation/filter_docs.pl +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/perl - -#Filters the phrase&cluster document set to retain only documents that correspond to words or morphs, i.e. not crossing word boundaries. - -#Usage: filter_docs.pl [mark] -# STDIN: data in the doc.txt format (i.e. phrase\t blahblah ), most likely from cdec extractor -# STDOUT: the matching subset, same format - -use utf8; -my $letter=qr/\p{L}\p{M}*/; # see http://www.regular-expressions.info/unicode.html - -my $morph=qr/$letter+/; - -my $m = "##"; # marker used to indicate morphemes -if ((scalar @ARGV) >= 1) { - $m = $ARGV[0]; - shift; -} -print STDERR "Using $m to filter for morphemes\n"; - -my $expr = qr/^($morph\Q$m\E)? ?(\Q$m\E$morph\Q$m\E)* ?(\Q$m\E$morph)?\t/; #\Q and \E bounded sections are escaped -while(<>) { - /$expr/ && print; -} diff --git a/gi/morf-segmentation/invalid_vocab.patterns b/gi/morf-segmentation/invalid_vocab.patterns deleted file mode 100644 index 473ce1b1..00000000 --- a/gi/morf-segmentation/invalid_vocab.patterns +++ /dev/null @@ -1,6 +0,0 @@ -[[:digit:]] -[] !"#$%&()*+,./:;<=>?@[\^_`{|}~] -^'$ --$ -^- -^$ diff --git a/gi/morf-segmentation/linestripper.py b/gi/morf-segmentation/linestripper.py deleted file mode 100755 index 04e9044a..00000000 --- a/gi/morf-segmentation/linestripper.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python - -import sys - -#linestripper file file maxlen [numlines] - -if len(sys.argv) < 3: - print "linestripper file1 file2 maxlen [numlines]" - print " outputs subset of file1 to stdout, ..of file2 to stderr" - sys.exit(1) - - -f1 = open(sys.argv[1],'r') -f2 = open(sys.argv[2],'r') - -maxlen=int(sys.argv[3]) -numlines = 0 - -if len(sys.argv) > 4: - numlines = int(sys.argv[4]) - -count=0 -for line1 in f1: - line2 = f2.readline() - - w1 = len(line1.strip().split()) - w2 = len(line2.strip().split()) - - if w1 <= maxlen and w2 <= maxlen: - count = count + 1 - sys.stdout.write(line1) - sys.stderr.write(line2) - - if numlines > 0 and count >= numlines: - break - -f1.close() -f2.close() - - diff --git a/gi/morf-segmentation/morf-pipeline.pl b/gi/morf-segmentation/morf-pipeline.pl deleted file mode 100755 index 46eb5b46..00000000 --- a/gi/morf-segmentation/morf-pipeline.pl +++ /dev/null @@ -1,486 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use File::Copy; - - -# Preprocessing pipeline to take care of word segmentation -# Learns a segmentation model for each/either side of the parallel corpus using all train/dev/test data -# Applies the segmentation where necessary. -# Learns word alignments on the preprocessed training data. -# Outputs script files used later to score output. - - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path cwd /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } - -use Getopt::Long "GetOptions"; - -my $GZIP = 'gzip'; -my $ZCAT = 'gunzip -c'; -my $SED = 'sed -e'; - -my $MORF_TRAIN = "$SCRIPT_DIR/morftrain.sh"; -my $MORF_SEGMENT = "$SCRIPT_DIR/morfsegment.py"; - -my $LINESTRIPPER = "$SCRIPT_DIR/linestripper.py"; -my $ALIGNER = "/export/ws10smt/software/berkeleyaligner/berkeleyaligner.jar"; -#java -d64 -Xmx10g -jar $ALIGNER ++word-align.conf >> aligner.log -assert_exec($MORF_TRAIN, $LINESTRIPPER, $MORF_SEGMENT, $ALIGNER); - -my $OUTPUT = './morfwork'; -my $PPL_SRC = 50; -my $PPL_TRG = 50; -my $MARKER = "#"; -my $MAX_WORDS = 40; -my $SENTENCES;# = 100000; -my $SPLIT_TYPE = ""; #possible values: s, t, st, or (empty string) -my $NAME_SHORTCUT; - -usage() unless &GetOptions('max_words=i' => \$MAX_WORDS, - 'output=s' => \$OUTPUT, - 'ppl_src=i' => \$PPL_SRC, - 'ppl_trg=i' => \$PPL_TRG, - 'sentences=i' => \$SENTENCES, - 'marker=s' => \$MARKER, - 'split=s' => \$SPLIT_TYPE, - 'get_name_only' => \$NAME_SHORTCUT, - ); - -usage() unless scalar @ARGV >= 2; - -my %CORPUS; # for (src,trg) it has (orig, name, filtered, final) - -$CORPUS{'src'}{'orig'} = $ARGV[0]; -open F, "<$CORPUS{'src'}{'orig'}" or die "Can't read $CORPUS{'src'}{'orig'}: $!"; close F; -$CORPUS{'src'}{'name'} = get_basename($CORPUS{'src'}{'orig'}); - -$CORPUS{'trg'}{'orig'} = $ARGV[1]; -open F, "<$CORPUS{'trg'}{'orig'}" or die "Can't read $CORPUS{'trg'}{'orig'}: $!"; close F; -$CORPUS{'trg'}{'name'} = get_basename($CORPUS{'trg'}{'orig'}); - -my %DEV; # for (src,trg) has (orig, final.split final.unsplit -if (@ARGV >= 4) { - $DEV{'src'}{'orig'} = $ARGV[2]; - open F, "<$DEV{'src'}{'orig'}" or die "Can't read $DEV{'src'}{'orig'}: $!"; close F; - $DEV{'src'}{'name'} = get_basename($DEV{'src'}{'orig'}); - $DEV{'trg'}{'orig'} = $ARGV[3]; - open F, "<$DEV{'trg'}{'orig'}" or die "Can't read $DEV{'trg'}{'orig'}: $!"; close F; - $DEV{'trg'}{'name'} = get_basename($DEV{'trg'}{'orig'}); -} - -my %TEST; # for (src,trg) has (orig, name) -if (@ARGV >= 6) { - $TEST{'src'}{'orig'} = $ARGV[4]; - open F, "<$TEST{'src'}{'orig'}" or die "Can't read $TEST{'src'}{'orig'}: $!"; close F; - $TEST{'src'}{'name'} = get_basename($TEST{'src'}{'orig'}); - $TEST{'trg'}{'orig'} = $ARGV[5]; - open F, "<$TEST{'trg'}{'orig'}" or die "Can't read $TEST{'trg'}{'orig'}: $!"; close F; - $TEST{'trg'}{'name'} = get_basename($TEST{'trg'}{'orig'}); -} - -my $SPLIT_SRC; #use these to check whether that part is being split -my $SPLIT_TRG; - -#OUTPUT WILL GO IN THESE -my $CORPUS_DIR = $OUTPUT . '/' . corpus_dir(); #subsampled corpus -my $MODEL_SRC_DIR = $OUTPUT . '/' . model_dir("src"); #splitting.. -my $MODEL_TRG_DIR = $OUTPUT . '/' . model_dir("trg"); # .. models -my $PROCESSED_DIR = $OUTPUT . '/' . processed_dir(); #segmented copora+alignments -my $ALIGNMENT_DIR = $PROCESSED_DIR . '/alignments'; - -$CORPUS{'src'}{'filtered'} = $CORPUS_DIR . "/$CORPUS{'src'}{'name'}"; -$CORPUS{'trg'}{'filtered'} = $CORPUS_DIR . "/$CORPUS{'trg'}{'name'}"; - -print STDERR "Output: $OUTPUT\n"; -print STDERR "Corpus: $CORPUS_DIR\n"; -print STDERR "Model-src: $MODEL_SRC_DIR\n"; -print STDERR "Model-trg: $MODEL_TRG_DIR\n"; -print STDERR "Finaldir: $PROCESSED_DIR\n"; - -safemkdir($OUTPUT) or die "Couldn't create output directory $OUTPUT: $!"; -safemkdir($CORPUS_DIR) or die "Couldn't create output directory $CORPUS_DIR: $!"; -filter_corpus(); - -safemkdir($PROCESSED_DIR); -safemkdir($ALIGNMENT_DIR); - -if ($SPLIT_SRC) { - safemkdir($MODEL_SRC_DIR) or die "Couldn't create output directory $MODEL_SRC_DIR: $!"; - learn_segmentation("src"); - apply_segmentation_side("src", $MODEL_SRC_DIR); -} - -#assume that unsplit hypotheses will be scored against an aritificially split target test set; thus obtain a target splitting model -#TODO: add a flag to override this behaviour -safemkdir($MODEL_TRG_DIR) or die "Couldn't create output directory $MODEL_TRG_DIR: $!"; -learn_segmentation("trg"); -$TEST{'trg'}{'finalunsplit'} = "$PROCESSED_DIR/$TEST{'trg'}{'name'}"; -copy($TEST{'trg'}{'orig'}, $TEST{'trg'}{'finalunsplit'}) or die "Could not copy unsegmented test set"; - -if ($SPLIT_TRG) { - apply_segmentation_side("trg", $MODEL_TRG_DIR); - } else { - $TEST{'trg'}{'finalsplit'} = "$PROCESSED_DIR/$TEST{'trg'}{'name'}.split"; - apply_segmentation_any($MODEL_TRG_DIR, $TEST{'trg'}{'finalunsplit'}, $TEST{'trg'}{'finalsplit'}); -} - -write_eval_sh("$PROCESSED_DIR/eval-devtest.sh"); - -#copy corpora if they haven't been put in place by splitting operations -place_missing_data_side('src'); -place_missing_data_side('trg'); - -do_align(); - -if ($CORPUS{'src'}{'orig'} && $DEV{'src'}{'orig'} && $TEST{'src'}{'orig'}) { - print STDERR "Putting the config file entry in $PROCESSED_DIR/exp.config\n"; -#format is: - # nlfr100k_unsplit /export/ws10smt/jan/nlfr/morfwork/s100k.w40.sp_0 corpus.nl-fr.al fr-3.lm.gz dev.nl dev.fr test2008.nl eval-devtest.sh - my $line = split_name() . " $PROCESSED_DIR corpus.src-trg.al LMFILE.lm.gz"; - $line = $line . " $DEV{'src'}{'name'} $DEV{'trg'}{'name'}"; - $line = $line . " " . get_basename($TEST{'src'}{$SPLIT_SRC ? "finalsplit" : "finalunsplit"}) . " eval-devtest.sh"; - safesystem("echo '$line' > $PROCESSED_DIR/exp.config"); -} - -system("date"); -print STDERR "All done. You now need to train a language model (if target split), put it in the right dir and update the config file.\n\n"; - -############################## BILINGUAL ################################### - -sub filter_corpus { - print STDERR "\n!!!FILTERING TRAINING COPRUS!!!\n"; - if ( -f $CORPUS{'src'}{'filtered'} && -f $CORPUS{'trg'}{'filtered'}) { - print STDERR "$CORPUS{'src'}{'filtered'} and $CORPUS{'trg'}{'filtered'} exist, reusing...\n"; - return; - } - my $args = "$CORPUS{'src'}{'orig'} $CORPUS{'trg'}{'orig'} $MAX_WORDS"; - if ($SENTENCES) { $args = $args . " $SENTENCES"; } - safesystem("$LINESTRIPPER $args 1> $CORPUS{'src'}{'filtered'} 2> $CORPUS{'trg'}{'filtered'}") or die "Failed to filter training corpus for length."; -} - -sub learn_segmentation -{ - my $WHICH = shift; - my $corpus; my $dev; my $test; my $moddir; my $ppl; - - $corpus = $CORPUS{$WHICH}{'filtered'}; - $dev = $DEV{$WHICH}{'orig'}; - $test = $TEST{$WHICH}{'orig'}; - - if ($WHICH eq "src") { - $moddir = $MODEL_SRC_DIR; - $ppl = $PPL_SRC; - } else { - $moddir = $MODEL_TRG_DIR; - $ppl = $PPL_TRG; - } - my $cmd = "cat $corpus"; - if ($dev) { $cmd = "$cmd $dev"; } - if ($test) { $cmd = "$cmd $test"; } - my $tmpfile = "$CORPUS_DIR/all.tmp.gz"; - safesystem("$cmd | $GZIP > $tmpfile") or die "Failed to concatenate data for model learning.."; - assert_marker($tmpfile); - - learn_segmentation_side($tmpfile, $moddir, $ppl, $WHICH); - safesystem("rm $tmpfile"); -} - -sub do_align { - print STDERR "\n!!!WORD ALIGNMENT!!!\n"; - system("date"); - - my $ALIGNMENTS = "$ALIGNMENT_DIR/training.align"; - if ( -f $ALIGNMENTS ) { - print STDERR "$ALIGNMENTS exists, reusing...\n"; - return; - } - my $conf_file = "$ALIGNMENT_DIR/word-align.conf"; - - #decorate training files with identifiers to stop the aligner from training on dev and test when rerun in future. - safesystem("cd $PROCESSED_DIR && ln -s $CORPUS{'src'}{'name'} corpus.src") or die "Failed to symlink: $!"; - safesystem("cd $PROCESSED_DIR && ln -s $CORPUS{'trg'}{'name'} corpus.trg") or die "Failed to symlink: $!"; - - write_wconf($conf_file, $PROCESSED_DIR); - system("java -d64 -Xmx24g -jar $ALIGNER ++$conf_file > $ALIGNMENT_DIR/aligner.log"); - - if (! -f $ALIGNMENTS) { die "Failed to run word alignment.";} - - my $cmd = "paste $PROCESSED_DIR/corpus.src $PROCESSED_DIR/corpus.trg $ALIGNMENTS"; - $cmd = $cmd . " | sed 's/\\t/ \|\|\| /g' > $PROCESSED_DIR/corpus.src-trg.al"; - safesystem($cmd) or die "Failed to paste into aligned corpus file."; - -} - -############################# MONOLINGUAL ################################# - -#copy the necessary data files that weren't place by segmentation -sub place_missing_data_side { - my $side = shift; - - ifne_copy($CORPUS{$side}{'filtered'}, "$PROCESSED_DIR/$CORPUS{$side}{'name'}") ; - - if ($DEV{$side}{'orig'} && ! -f "$PROCESSED_DIR/$DEV{$side}{'name'}") { - $DEV{$side}{'final'} = "$PROCESSED_DIR/$DEV{$side}{'name'}"; - copy($DEV{$side}{'orig'}, $DEV{$side}{'final'}) or die "Copy failed: $!"; - } - - if ($TEST{$side}{'orig'} && ! -f "$PROCESSED_DIR/$TEST{$side}{'name'}" && ! $TEST{$side}{'finalunsplit'}) { - $TEST{$side}{'finalunsplit'} = "$PROCESSED_DIR/$TEST{$side}{'name'}"; - copy($TEST{$side}{'orig'}, $TEST{$side}{'finalunsplit'}) or die "Copy failed: $!"; - } - -} - -sub apply_segmentation_side { - my ($side, $moddir) = @_; - - print STDERR "\n!!!APPLYING SEGMENTATION MODEL ($side)!!!\n"; - apply_segmentation_any($moddir, $CORPUS{$side}{'filtered'}, "$PROCESSED_DIR/$CORPUS{$side}{'name'}"); - if ($DEV{$side}{'orig'}) { - $DEV{$side}{'final'} = "$PROCESSED_DIR/$DEV{$side}{'name'}"; - apply_segmentation_any($moddir, $DEV{$side}{'orig'}, "$DEV{$side}{'final'}"); - } - if ($TEST{$side}{'orig'}) { - $TEST{$side}{'finalsplit'} = "$PROCESSED_DIR/$TEST{$side}{'name'}.split"; - apply_segmentation_any($moddir, $TEST{$side}{'orig'}, $TEST{$side}{'finalsplit'} ); - } - -} - -sub learn_segmentation_side { - my($INPUT_FILE, $SEGOUT_DIR, $PPL, $LANG) = @_; - - print STDERR "\n!!!LEARNING SEGMENTATION MODEL ($LANG)!!!\n"; - system("date"); - my $SEG_FILE = $SEGOUT_DIR . "/segmentation.ready"; - if ( -f $SEG_FILE) { - print STDERR "$SEG_FILE exists, reusing...\n"; - return; - } - my $cmd = "$MORF_TRAIN $INPUT_FILE $SEGOUT_DIR $PPL \"$MARKER\""; - safesystem($cmd) or die "Failed to learn segmentation model"; -} - -sub apply_segmentation_any { - my($moddir, $datfile, $outfile) = @_; - if ( -f $outfile) { - print STDERR "$outfile exists, reusing...\n"; - return; - } - - my $args = "$moddir/inputvocab.gz $moddir/segmentation.ready \"$MARKER\""; - safesystem("cat $datfile | $MORF_SEGMENT $args &> $outfile") or die "Could not segment $datfile"; -} - -##################### PATH FUNCTIONS ########################## - -sub beautify_numlines { - return ($SENTENCES ? $SENTENCES : "_all"); -} - -sub corpus_dir { - return "s" . beautify_numlines() . ".w" . $MAX_WORDS; -} - -sub model_dir { - my $lang = shift; - if ($lang eq "src") { - return corpus_dir() . ".PPL" . $PPL_SRC . ".src"; - } elsif ($lang eq "trg") { - return corpus_dir() . ".PPL" . $PPL_TRG . ".trg"; - } else { - return "PPLundef"; - } -} - -sub processed_dir { - return corpus_dir() . "." . split_name(); -} - -########################## HELPER FUNCTIONS ############################ - -sub ifne_copy { - my ($src, $dest) = @_; - if (! -f $dest) { - copy($src, $dest) or die "Copy failed: $!"; - } -} - -sub split_name { - #parses SPLIT_TYPE, which can have the following values - # t|s|ts|st (last 2 are equiv) - # or is undefined when no splitting is done - my $name = ""; - - if ($SPLIT_TYPE) { - $SPLIT_SRC = lc($SPLIT_TYPE) =~ /s/; - $SPLIT_TRG = lc($SPLIT_TYPE) =~ /t/; - $name = $name . ($SPLIT_SRC ? $PPL_SRC : "0"); - $name = $name . "_" . ($SPLIT_TRG ? $PPL_TRG : "0"); - } else { - #no splitting - $name = "0"; - } - - return "sp_" . $name; - -} - -sub usage { - print <<EOT; - -Usage: $0 [OPTIONS] corpus.src corpus.trg [dev.src dev.trg [test.src test.trg]] - -Learns a segmentation model and splits up corpora as necessary. Word alignments are trained on a specified subset of the training corpus. - -EOT - exit 1; -}; - -sub safemkdir { - my $dir = shift; - if (-d $dir) { return 1; } - return mkdir($dir); -} - -sub assert_exec { - my @files = @_; - for my $file (@files) { - die "Can't find $file - did you run make?\n" unless -e $file; - die "Can't execute $file" unless -e $file; - } -}; -sub safesystem { - print STDERR "Executing: @_\n"; - system(@_); - if ($? == -1) { - print STDERR "ERROR: Failed to execute: @_\n $!\n"; - exit(1); - } - elsif ($? & 127) { - printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - exit(1); - } - else { - my $exitcode = $? >> 8; - print STDERR "Exit code: $exitcode\n" if $exitcode; - return ! $exitcode; - } -} - -sub get_basename -{ - my $x = shift; - $x = `basename $x`; - $x =~ s/\n//; - return $x; -} - -sub assert_marker { - my $file = shift; - my $result = `zcat $file| grep '$MARKER' | wc -l` or die "Cannot read $file: $!"; - print $result; - if (scalar($result) != 0) { die "Data contains marker '$MARKER'; use something else.";} -} -########################### Dynamic config files ############################## - -sub write_wconf { - my ($filename, $train_dir) = @_; - open WCONF, ">$filename" or die "Can't write $filename: $!"; - - print WCONF <<EOT; -## ---------------------- -## This is an example training script for the Berkeley -## word aligner. In this configuration it uses two HMM -## alignment models trained jointly and then decoded -## using the competitive thresholding heuristic. - -########################################## -# Training: Defines the training regimen -########################################## -forwardModels MODEL1 HMM -reverseModels MODEL1 HMM -mode JOINT JOINT -iters 5 5 - -############################################### -# Execution: Controls output and program flow -############################################### -execDir $ALIGNMENT_DIR -create -overwriteExecDir -saveParams true -numThreads 1 -msPerLine 10000 -alignTraining - -################# -# Language/Data -################# -foreignSuffix src -englishSuffix trg - -# Choose the training sources, which can either be directories or files that list files/directories -trainSources $train_dir/ -#trainSources $train_dir/sources -testSources -sentences MAX - -################# -# 1-best output -################# -competitiveThresholding - -EOT - close WCONF; -} - -sub write_eval_sh -{ - my ($filename) = @_; - open EVALFILE, ">$filename" or die "Can't write $filename: $!"; - - print EVALFILE <<EOT; -#!/bin/bash - -EVAL_MAIN=/export/ws10smt/data/eval.sh -marker="$MARKER" -EOT - - if ($SPLIT_TRG) { - print EVALFILE <<EOT; -echo "OUTPUT EVALUATION" -echo "-----------------" -\$EVAL_MAIN "\$1" $TEST{'trg'}{'finalsplit'} - -echo "RECOMBINED OUTPUT EVALUATION" -echo "----------------------------" -cat "\$1" | sed -e "s/\$marker \$marker//g" -e "s/\$marker//g" > "\$1.recombined" - -\$EVAL_MAIN "\$1.recombined" $TEST{'trg'}{'finalunsplit'} -EOT - - } else { - print EVALFILE <<EOT; -echo "ARTIFICIAL SPLIT EVALUATION" -echo "--------------------------" - -#split the output translation -cat "\$1" | $MORF_SEGMENT $MODEL_TRG_DIR/inputvocab.gz $MODEL_TRG_DIR/segmentation.ready "\$MARKER" > "\$1.split" - -\$EVAL_MAIN "\$1.split" $TEST{'trg'}{'finalsplit'} - -echo "DIRECT EVALUATION" -echo "--------------------------" -\$EVAL_MAIN "\$1" $TEST{'trg'}{'finalunsplit'} - -EOT - - } - close EVALFILE; - -} - - - - diff --git a/gi/morf-segmentation/morfsegment.py b/gi/morf-segmentation/morfsegment.py deleted file mode 100755 index 85b9d4fb..00000000 --- a/gi/morf-segmentation/morfsegment.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/python - -import sys -import gzip - -#usage: morfsegment.py inputvocab.gz segmentation.ready -# stdin: the data to segment -# stdout: the segmented data - -if len(sys.argv) < 3: - print "usage: morfsegment.py inputvocab.gz segmentation.ready [marker]" - print " stdin: the data to segment" - print " stdout: the segmented data" - sys.exit() - -#read index: -split_index={} - -marker="##" - -if len(sys.argv) > 3: - marker=sys.argv[3] - -word_vocab=gzip.open(sys.argv[1], 'rb') #inputvocab.gz -seg_vocab=open(sys.argv[2], 'r') #segm.ready.. - -for seg in seg_vocab: - #seg = ver# #wonder\n - #wordline = 1 verwonder\n - word = word_vocab.readline().strip().split(' ') - assert(len(word) == 2) - word = word[1] - seg=seg.strip() - - if seg != word: - split_index[word] = seg - -word_vocab.close() -seg_vocab.close() - -for line in sys.stdin: - words = line.strip().split() - - newsent = [] - for word in words: - splitword = split_index.get(word, word) - newsent.append(splitword) - - print ' '.join(newsent) - diff --git a/gi/morf-segmentation/morftrain.sh b/gi/morf-segmentation/morftrain.sh deleted file mode 100755 index 9004922f..00000000 --- a/gi/morf-segmentation/morftrain.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash - -if [[ $# -lt 3 ]]; then - echo "Trains a morfessor model and places the result in writedir" - echo - echo "Usage: `basename $0` corpus_input_file writedir [PPL] [marker] [lines]" - echo -e "\tcorpus_input_file contains a sentence per line." - exit 1 -fi - -MORFESSOR_DIR="/export/ws10smt/software/morfessor_catmap0.9.2" -SCRIPT_DIR=$(dirname `readlink -f $0`) - -MORFBINDIR="$MORFESSOR_DIR/bin" -MORFMAKEFILE_TRAIN="$MORFESSOR_DIR/train/Makefile" -VOCABEXT="$SCRIPT_DIR/vocabextractor.sh" - -MARKER="#" - -if [[ ! -f $VOCABEXT ]]; then - echo "$VOCABEXT doesn't exist!" - exit 1 -fi -if [[ ! -f $MORFMAKEFILE_TRAIN ]]; then - echo "$MORFMAKEFILE_TRAIN doesn't exist!" - exit 1 -fi - - -CORPUS="$1" -WRITETODIR=$2 - -if [[ ! -f $CORPUS ]]; then - echo "$CORPUS doesn't exist!" - exit 1 -fi - -PPL=10 -LINES=0 -if [[ $# -gt 2 ]]; then - PPL=$3 -fi -if [[ $# -gt 3 ]]; then - MARKER="$4" -fi -if [[ $# -gt 4 ]]; then - LINES=$5 -fi - -mkdir -p $WRITETODIR - -#extract vocabulary to train on -echo "Extracting vocabulary..." -if [[ -f $WRITETODIR/inputvocab.gz ]]; then - echo " ....$WRITETODIR/inputvocab.gz exists, reusing." -else - if [[ $LINES -gt 0 ]]; then - $VOCABEXT $CORPUS $LINES | gzip > $WRITETODIR/inputvocab.gz - else - $VOCABEXT $CORPUS | gzip > $WRITETODIR/inputvocab.gz - fi -fi - - -#train it -echo "Training morf model..." -if [[ -f $WRITETODIR/segmentation.final.gz ]]; then - echo " ....$WRITETODIR/segmentation.final.gz exists, reusing.." -else - OLDPWD=`pwd` - cd $WRITETODIR - - #put the training Makefile in place, with appropriate modifications - sed -e "s/^GZIPPEDINPUTDATA = .*$/GZIPPEDINPUTDATA = inputvocab.gz/" \ - -e "s/^PPLTHRESH = .*$/PPLTHRESH = $PPL/" \ - -e "s;^BINDIR = .*$;BINDIR = $MORFBINDIR;" \ - $MORFMAKEFILE_TRAIN > ./Makefile - - date - make > ./trainmorf.log 2>&1 - cd $OLDPWD - - - echo "Post processing..." - #remove comments, counts and morph types - #mark morphs - - if [[ ! -f $WRITETODIR/segmentation.final.gz ]]; then - echo "Failed to learn segmentation model: $WRITETODIR/segmentation.final.gz not written" - exit 1 - fi - - zcat $WRITETODIR/segmentation.final.gz | \ - awk '$1 !~ /^#/ {print}' | \ - cut -d ' ' --complement -f 1 | \ - sed -e "s/\/...//g" -e "s/ + /$MARKER $MARKER/g" \ - > $WRITETODIR/segmentation.ready - - if [[ ! -f $WRITETODIR/segmentation.ready ]]; then - echo "Failed to learn segmentation model: $WRITETODIR/segmentation.final.gz not written" - exit 1 - fi - - - - echo "Done training." - date -fi -echo "Segmentation model is $WRITETODIR/segmentation.ready." - diff --git a/gi/morf-segmentation/vocabextractor.sh b/gi/morf-segmentation/vocabextractor.sh deleted file mode 100755 index 00ae7109..00000000 --- a/gi/morf-segmentation/vocabextractor.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -d=$(dirname `readlink -f $0`) -if [ $# -lt 1 ]; then - echo "Extracts unique words and their frequencies from a subset of a corpus." - echo - echo "Usage: `basename $0` input_file [number_of_lines] > output_file" - echo -e "\tinput_file contains a sentence per line." - echo - echo "Script also removes words from the vocabulary if they contain a digit or a special character. Output is printed to stdout in a format suitable for use with Morfessor." - echo - exit -fi - -srcname=$1 -reallen=0 - -if [[ $# -gt 1 ]]; then - reallen=$2 -fi - -pattern_file=$d/invalid_vocab.patterns - -if [[ ! -f $pattern_file ]]; then - echo "Pattern file missing" - exit 1 -fi - -#this awk strips entries from the vocabulary if they contain invalid characters -#invalid characters are digits and punctuation marks, and words beginning or ending with a dash -#uniq -c extracts the unique words and counts the occurrences - -if [[ $reallen -eq 0 ]]; then - #when a zero is passed, use the whole file - zcat -f $srcname | sed 's/ /\n/g' | egrep -v -f $pattern_file | sort | uniq -c | sed 's/^ *//' - -else - zcat -f $srcname | head -n $reallen | sed 's/ /\n/g' | egrep -v -f $pattern_file | sort | uniq -c | sed 's/^ *//' -fi - diff --git a/gi/pf/Makefile.am b/gi/pf/Makefile.am deleted file mode 100644 index 86f8e07b..00000000 --- a/gi/pf/Makefile.am +++ /dev/null @@ -1,44 +0,0 @@ -bin_PROGRAMS = cbgi brat dpnaive pfbrat pfdist itg pfnaive condnaive align-lexonly-pyp learn_cfg pyp_lm nuisance_test align-tl pf_test bayes_lattice_score - -noinst_LIBRARIES = libpf.a - -libpf_a_SOURCES = base_distributions.cc reachability.cc cfg_wfst_composer.cc corpus.cc unigrams.cc ngram_base.cc transliterations.cc backward.cc hpyp_tm.cc pyp_tm.cc - -bayes_lattice_score_SOURCES = bayes_lattice_score.cc -bayes_lattice_score_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -pf_test_SOURCES = pf_test.cc -pf_test_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -nuisance_test_SOURCES = nuisance_test.cc -nuisance_test_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -align_lexonly_pyp_SOURCES = align-lexonly-pyp.cc -align_lexonly_pyp_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -align_tl_SOURCES = align-tl.cc -align_tl_LDADD = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a -lz - -itg_SOURCES = itg.cc - -pyp_lm_SOURCES = pyp_lm.cc - -learn_cfg_SOURCES = learn_cfg.cc - -condnaive_SOURCES = condnaive.cc - -dpnaive_SOURCES = dpnaive.cc - -pfdist_SOURCES = pfdist.cc - -pfnaive_SOURCES = pfnaive.cc - -cbgi_SOURCES = cbgi.cc - -brat_SOURCES = brat.cc - -pfbrat_SOURCES = pfbrat.cc - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -funroll-loops -I$(top_srcdir)/utils $(GTEST_CPPFLAGS) -I$(top_srcdir)/decoder -I$(top_srcdir)/klm - -AM_LDFLAGS = libpf.a $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/klm/lm/libklm.a $(top_srcdir)/klm/util/libklm_util.a $(top_srcdir)/utils/libutils.a -lz diff --git a/gi/pf/README b/gi/pf/README deleted file mode 100644 index 62e47541..00000000 --- a/gi/pf/README +++ /dev/null @@ -1,2 +0,0 @@ -Experimental Bayesian alignment tools. Nothing to see here. - diff --git a/gi/pf/align-lexonly-pyp.cc b/gi/pf/align-lexonly-pyp.cc deleted file mode 100644 index e7509f57..00000000 --- a/gi/pf/align-lexonly-pyp.cc +++ /dev/null @@ -1,243 +0,0 @@ -#include <iostream> -#include <queue> - -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "tdict.h" -#include "stringlib.h" -#include "filelib.h" -#include "array2d.h" -#include "sampler.h" -#include "corpus.h" -#include "pyp_tm.h" -#include "hpyp_tm.h" -#include "quasi_model2.h" - -using namespace std; -namespace po = boost::program_options; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("infer_alignment_hyperparameters,I", "Infer alpha and p_null, otherwise fixed values will be assumed") - ("p_null,0", po::value<double>()->default_value(0.08), "probability of aligning to null") - ("align_alpha,a", po::value<double>()->default_value(4.0), "how 'tight' is the bias toward be along the diagonal?") - ("input,i",po::value<string>(),"Read parallel data from") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -MT19937* prng; - -struct LexicalAlignment { - unsigned char src_index; - bool is_transliteration; - vector<pair<short, short> > derivation; -}; - -struct AlignedSentencePair { - vector<WordID> src; - vector<WordID> trg; - vector<LexicalAlignment> a; - Array2D<short> posterior; -}; - -template <class LexicalTranslationModel> -struct Aligner { - Aligner(const vector<vector<WordID> >& lets, - int vocab_size, - int num_letters, - const po::variables_map& conf, - vector<AlignedSentencePair>* c) : - corpus(*c), - paj_model(conf["align_alpha"].as<double>(), conf["p_null"].as<double>()), - infer_paj(conf.count("infer_alignment_hyperparameters") > 0), - model(lets, vocab_size, num_letters), - kNULL(TD::Convert("NULL")) { - assert(lets[kNULL].size() == 0); - } - - vector<AlignedSentencePair>& corpus; - QuasiModel2 paj_model; - const bool infer_paj; - LexicalTranslationModel model; - const WordID kNULL; - - void ResampleHyperparameters() { - model.ResampleHyperparameters(prng); - if (infer_paj) paj_model.ResampleHyperparameters(prng); - } - - void InitializeRandom() { - cerr << "Initializing with random alignments ...\n"; - for (unsigned i = 0; i < corpus.size(); ++i) { - AlignedSentencePair& asp = corpus[i]; - asp.a.resize(asp.trg.size()); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - unsigned char& a_j = asp.a[j].src_index; - a_j = prng->next() * (1 + asp.src.size()); - const WordID f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - model.Increment(f_a_j, asp.trg[j], &*prng); - paj_model.Increment(a_j, j, asp.src.size(), asp.trg.size()); - } - } - cerr << "Corpus intialized randomly." << endl; - cerr << "LLH = " << Likelihood() << " \t(Amodel=" << paj_model.Likelihood() - << " TModel=" << model.Likelihood() << ") contexts=" << model.UniqueConditioningContexts() << endl; - } - - void ResampleCorpus() { - for (unsigned i = 0; i < corpus.size(); ++i) { - AlignedSentencePair& asp = corpus[i]; - SampleSet<prob_t> ss; ss.resize(asp.src.size() + 1); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - unsigned char& a_j = asp.a[j].src_index; - const WordID e_j = asp.trg[j]; - WordID f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - model.Decrement(f_a_j, e_j, prng); - paj_model.Decrement(a_j, j, asp.src.size(), asp.trg.size()); - - for (unsigned prop_a_j = 0; prop_a_j <= asp.src.size(); ++prop_a_j) { - const WordID prop_f = (prop_a_j ? asp.src[prop_a_j - 1] : kNULL); - ss[prop_a_j] = model.Prob(prop_f, e_j); - ss[prop_a_j] *= paj_model.Prob(prop_a_j, j, asp.src.size(), asp.trg.size()); - } - a_j = prng->SelectSample(ss); - f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - model.Increment(f_a_j, e_j, prng); - paj_model.Increment(a_j, j, asp.src.size(), asp.trg.size()); - } - } - } - - prob_t Likelihood() const { - return model.Likelihood() * paj_model.Likelihood(); - } -}; - -void ExtractLetters(const set<WordID>& v, vector<vector<WordID> >* l, set<WordID>* letset = NULL) { - for (set<WordID>::const_iterator it = v.begin(); it != v.end(); ++it) { - vector<WordID>& letters = (*l)[*it]; - if (letters.size()) continue; // if e and f have the same word - - const string& w = TD::Convert(*it); - - size_t cur = 0; - while (cur < w.size()) { - const size_t len = UTF8Len(w[cur]); - letters.push_back(TD::Convert(w.substr(cur, len))); - if (letset) letset->insert(letters.back()); - cur += len; - } - } -} - -void Debug(const AlignedSentencePair& asp) { - cerr << TD::GetString(asp.src) << endl << TD::GetString(asp.trg) << endl; - Array2D<bool> a(asp.src.size(), asp.trg.size()); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - assert(asp.a[j].src_index <= asp.src.size()); - if (asp.a[j].src_index) a(asp.a[j].src_index - 1, j) = true; - } - cerr << a << endl; -} - -void AddSample(AlignedSentencePair* asp) { - for (unsigned j = 0; j < asp->trg.size(); ++j) - asp->posterior(asp->a[j].src_index, j)++; -} - -void WriteAlignments(const AlignedSentencePair& asp) { - bool first = true; - for (unsigned j = 0; j < asp.trg.size(); ++j) { - int src_index = -1; - int mc = -1; - for (unsigned i = 0; i <= asp.src.size(); ++i) { - if (asp.posterior(i, j) > mc) { - mc = asp.posterior(i, j); - src_index = i; - } - } - - if (src_index) { - if (first) first = false; else cout << ' '; - cout << (src_index - 1) << '-' << j; - } - } - cout << endl; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - - if (conf.count("random_seed")) - prng = new MT19937(conf["random_seed"].as<uint32_t>()); - else - prng = new MT19937; - - vector<vector<int> > corpuse, corpusf; - set<int> vocabe, vocabf; - corpus::ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - vector<AlignedSentencePair> corpus(corpuse.size()); - for (unsigned i = 0; i < corpuse.size(); ++i) { - corpus[i].src.swap(corpusf[i]); - corpus[i].trg.swap(corpuse[i]); - corpus[i].posterior.resize(corpus[i].src.size() + 1, corpus[i].trg.size()); - } - corpusf.clear(); corpuse.clear(); - - vocabf.insert(TD::Convert("NULL")); - vector<vector<WordID> > letters(TD::NumWords()); - set<WordID> letset; - ExtractLetters(vocabe, &letters, &letset); - ExtractLetters(vocabf, &letters, NULL); - letters[TD::Convert("NULL")].clear(); - - //Aligner<PYPLexicalTranslation> aligner(letters, vocabe.size(), letset.size(), conf, &corpus); - Aligner<HPYPLexicalTranslation> aligner(letters, vocabe.size(), letset.size(), conf, &corpus); - aligner.InitializeRandom(); - - const unsigned samples = conf["samples"].as<unsigned>(); - for (int i = 0; i < samples; ++i) { - for (int j = 65; j < 67; ++j) Debug(corpus[j]); - if (i % 10 == 9) { - aligner.ResampleHyperparameters(); - cerr << "LLH = " << aligner.Likelihood() << " \t(Amodel=" << aligner.paj_model.Likelihood() - << " TModel=" << aligner.model.Likelihood() << ") contexts=" << aligner.model.UniqueConditioningContexts() << endl; - } - aligner.ResampleCorpus(); - if (i > (samples / 5) && (i % 6 == 5)) for (int j = 0; j < corpus.size(); ++j) AddSample(&corpus[j]); - } - for (unsigned i = 0; i < corpus.size(); ++i) - WriteAlignments(corpus[i]); - aligner.model.Summary(); - - return 0; -} diff --git a/gi/pf/align-tl.cc b/gi/pf/align-tl.cc deleted file mode 100644 index f6608f1d..00000000 --- a/gi/pf/align-tl.cc +++ /dev/null @@ -1,339 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/multi_array.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "backward.h" -#include "array2d.h" -#include "base_distributions.h" -#include "monotonic_pseg.h" -#include "conditional_pseg.h" -#include "trule.h" -#include "tdict.h" -#include "stringlib.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "mfcr.h" -#include "corpus.h" -#include "ngram_base.h" -#include "transliterations.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("input,i",po::value<string>(),"Read parallel data from") - ("s2t", po::value<string>(), "character level source-to-target prior transliteration probabilities") - ("t2s", po::value<string>(), "character level target-to-source prior transliteration probabilities") - ("max_src_chunk", po::value<unsigned>()->default_value(4), "Maximum size of translitered chunk in source") - ("max_trg_chunk", po::value<unsigned>()->default_value(4), "Maximum size of translitered chunk in target") - ("expected_src_to_trg_ratio", po::value<double>()->default_value(1.0), "If a word is transliterated, what is the expected length ratio from source to target?") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -boost::shared_ptr<MT19937> prng; - -struct LexicalAlignment { - unsigned char src_index; - bool is_transliteration; - vector<pair<short, short> > derivation; -}; - -struct AlignedSentencePair { - vector<WordID> src; - vector<WordID> trg; - vector<LexicalAlignment> a; - Array2D<short> posterior; -}; - -struct HierarchicalWordBase { - explicit HierarchicalWordBase(const unsigned vocab_e_size) : - base(prob_t::One()), r(1,1,1,1,0.66,50.0), u0(-log(vocab_e_size)), l(1,prob_t::One()), v(1, prob_t::Zero()) {} - - void ResampleHyperparameters(MT19937* rng) { - r.resample_hyperparameters(rng); - } - - inline double logp0(const vector<WordID>& s) const { - return Md::log_poisson(s.size(), 7.5) + s.size() * u0; - } - - // return p0 of rule.e_ - prob_t operator()(const TRule& rule) const { - v[0].logeq(logp0(rule.e_)); - return r.prob(rule.e_, v.begin(), l.begin()); - } - - void Increment(const TRule& rule) { - v[0].logeq(logp0(rule.e_)); - if (r.increment(rule.e_, v.begin(), l.begin(), &*prng).count) { - base *= v[0] * l[0]; - } - } - - void Decrement(const TRule& rule) { - if (r.decrement(rule.e_, &*prng).count) { - base /= prob_t(exp(logp0(rule.e_))); - } - } - - prob_t Likelihood() const { - prob_t p; p.logeq(r.log_crp_prob()); - p *= base; - return p; - } - - void Summary() const { - cerr << "NUMBER OF CUSTOMERS: " << r.num_customers() << " (d=" << r.discount() << ",s=" << r.strength() << ')' << endl; - for (MFCR<1,vector<WordID> >::const_iterator it = r.begin(); it != r.end(); ++it) - cerr << " " << it->second.total_dish_count_ << " (on " << it->second.table_counts_.size() << " tables) " << TD::GetString(it->first) << endl; - } - - prob_t base; - MFCR<1,vector<WordID> > r; - const double u0; - const vector<prob_t> l; - mutable vector<prob_t> v; -}; - -struct BasicLexicalAlignment { - explicit BasicLexicalAlignment(const vector<vector<WordID> >& lets, - const unsigned words_e, - const unsigned letters_e, - vector<AlignedSentencePair>* corp) : - letters(lets), - corpus(*corp), - //up0(words_e), - //up0("en.chars.1gram", letters_e), - //up0("en.words.1gram"), - up0(letters_e), - //up0("en.chars.2gram"), - tmodel(up0) { - } - - void InstantiateRule(const WordID src, - const WordID trg, - TRule* rule) const { - static const WordID kX = TD::Convert("X") * -1; - rule->lhs_ = kX; - rule->e_ = letters[trg]; - rule->f_ = letters[src]; - } - - void InitializeRandom() { - const WordID kNULL = TD::Convert("NULL"); - cerr << "Initializing with random alignments ...\n"; - for (unsigned i = 0; i < corpus.size(); ++i) { - AlignedSentencePair& asp = corpus[i]; - asp.a.resize(asp.trg.size()); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - const unsigned char a_j = prng->next() * (1 + asp.src.size()); - const WordID f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - TRule r; - InstantiateRule(f_a_j, asp.trg[j], &r); - asp.a[j].is_transliteration = false; - asp.a[j].src_index = a_j; - if (tmodel.IncrementRule(r, &*prng)) - up0.Increment(r); - } - } - cerr << " LLH = " << Likelihood() << endl; - } - - prob_t Likelihood() const { - prob_t p = tmodel.Likelihood(); - p *= up0.Likelihood(); - return p; - } - - void ResampleHyperparemeters() { - tmodel.ResampleHyperparameters(&*prng); - up0.ResampleHyperparameters(&*prng); - cerr << " (base d=" << up0.r.discount() << ",s=" << up0.r.strength() << ")\n"; - } - - void ResampleCorpus(); - - const vector<vector<WordID> >& letters; // spelling dictionary - vector<AlignedSentencePair>& corpus; - //PhraseConditionalUninformativeBase up0; - //PhraseConditionalUninformativeUnigramBase up0; - //UnigramWordBase up0; - //HierarchicalUnigramBase up0; - HierarchicalWordBase up0; - //CompletelyUniformBase up0; - //FixedNgramBase up0; - //ConditionalTranslationModel<PhraseConditionalUninformativeBase> tmodel; - //ConditionalTranslationModel<PhraseConditionalUninformativeUnigramBase> tmodel; - //ConditionalTranslationModel<UnigramWordBase> tmodel; - //ConditionalTranslationModel<HierarchicalUnigramBase> tmodel; - MConditionalTranslationModel<HierarchicalWordBase> tmodel; - //ConditionalTranslationModel<FixedNgramBase> tmodel; - //ConditionalTranslationModel<CompletelyUniformBase> tmodel; -}; - -void BasicLexicalAlignment::ResampleCorpus() { - static const WordID kNULL = TD::Convert("NULL"); - for (unsigned i = 0; i < corpus.size(); ++i) { - AlignedSentencePair& asp = corpus[i]; - SampleSet<prob_t> ss; ss.resize(asp.src.size() + 1); - for (unsigned j = 0; j < asp.trg.size(); ++j) { - TRule r; - unsigned char& a_j = asp.a[j].src_index; - WordID f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - InstantiateRule(f_a_j, asp.trg[j], &r); - if (tmodel.DecrementRule(r, &*prng)) - up0.Decrement(r); - - for (unsigned prop_a_j = 0; prop_a_j <= asp.src.size(); ++prop_a_j) { - const WordID prop_f = (prop_a_j ? asp.src[prop_a_j - 1] : kNULL); - InstantiateRule(prop_f, asp.trg[j], &r); - ss[prop_a_j] = tmodel.RuleProbability(r); - } - a_j = prng->SelectSample(ss); - f_a_j = (a_j ? asp.src[a_j - 1] : kNULL); - InstantiateRule(f_a_j, asp.trg[j], &r); - if (tmodel.IncrementRule(r, &*prng)) - up0.Increment(r); - } - } - cerr << " LLH = " << Likelihood() << endl; -} - -void ExtractLetters(const set<WordID>& v, vector<vector<WordID> >* l, set<WordID>* letset = NULL) { - for (set<WordID>::const_iterator it = v.begin(); it != v.end(); ++it) { - vector<WordID>& letters = (*l)[*it]; - if (letters.size()) continue; // if e and f have the same word - - const string& w = TD::Convert(*it); - - size_t cur = 0; - while (cur < w.size()) { - const size_t len = UTF8Len(w[cur]); - letters.push_back(TD::Convert(w.substr(cur, len))); - if (letset) letset->insert(letters.back()); - cur += len; - } - } -} - -void Debug(const AlignedSentencePair& asp) { - cerr << TD::GetString(asp.src) << endl << TD::GetString(asp.trg) << endl; - Array2D<bool> a(asp.src.size(), asp.trg.size()); - for (unsigned j = 0; j < asp.trg.size(); ++j) - if (asp.a[j].src_index) a(asp.a[j].src_index - 1, j) = true; - cerr << a << endl; -} - -void AddSample(AlignedSentencePair* asp) { - for (unsigned j = 0; j < asp->trg.size(); ++j) - asp->posterior(asp->a[j].src_index, j)++; -} - -void WriteAlignments(const AlignedSentencePair& asp) { - bool first = true; - for (unsigned j = 0; j < asp.trg.size(); ++j) { - int src_index = -1; - int mc = -1; - for (unsigned i = 0; i <= asp.src.size(); ++i) { - if (asp.posterior(i, j) > mc) { - mc = asp.posterior(i, j); - src_index = i; - } - } - - if (src_index) { - if (first) first = false; else cout << ' '; - cout << (src_index - 1) << '-' << j; - } - } - cout << endl; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); -// MT19937& rng = *prng; - - vector<vector<int> > corpuse, corpusf; - set<int> vocabe, vocabf; - corpus::ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - vector<AlignedSentencePair> corpus(corpuse.size()); - for (unsigned i = 0; i < corpuse.size(); ++i) { - corpus[i].src.swap(corpusf[i]); - corpus[i].trg.swap(corpuse[i]); - corpus[i].posterior.resize(corpus[i].src.size() + 1, corpus[i].trg.size()); - } - corpusf.clear(); corpuse.clear(); - - vocabf.insert(TD::Convert("NULL")); - vector<vector<WordID> > letters(TD::NumWords() + 1); - set<WordID> letset; - ExtractLetters(vocabe, &letters, &letset); - ExtractLetters(vocabf, &letters, NULL); - letters[TD::Convert("NULL")].clear(); - - // TODO configure this - const int max_src_chunk = conf["max_src_chunk"].as<unsigned>(); - const int max_trg_chunk = conf["max_trg_chunk"].as<unsigned>(); - const double s2t_rat = conf["expected_src_to_trg_ratio"].as<double>(); - const BackwardEstimator be(conf["s2t"].as<string>(), conf["t2s"].as<string>()); - Transliterations tl(max_src_chunk, max_trg_chunk, s2t_rat, be); - - cerr << "Initializing transliteration graph structures ...\n"; - for (int i = 0; i < corpus.size(); ++i) { - const vector<int>& src = corpus[i].src; - const vector<int>& trg = corpus[i].trg; - for (int j = 0; j < src.size(); ++j) { - const vector<int>& src_let = letters[src[j]]; - for (int k = 0; k < trg.size(); ++k) { - const vector<int>& trg_let = letters[trg[k]]; - tl.Initialize(src[j], src_let, trg[k], trg_let); - //if (src_let.size() < min_trans_src) - // tl.Forbid(src[j], src_let, trg[k], trg_let); - } - } - } - cerr << endl; - tl.GraphSummary(); - - return 0; -} diff --git a/gi/pf/backward.cc b/gi/pf/backward.cc deleted file mode 100644 index b92629fd..00000000 --- a/gi/pf/backward.cc +++ /dev/null @@ -1,89 +0,0 @@ -#include "backward.h" - -#include <queue> -#include <utility> - -#include "array2d.h" -#include "reachability.h" -#include "base_distributions.h" - -using namespace std; - -BackwardEstimator::BackwardEstimator(const string& s2t, - const string& t2s) : m1(new Model1(s2t)), m1inv(new Model1(t2s)) {} - -BackwardEstimator::~BackwardEstimator() { - delete m1; m1 = NULL; - delete m1inv; m1inv = NULL; -} - -float BackwardEstimator::ComputeBackwardProb(const std::vector<WordID>& src, - const std::vector<WordID>& trg, - unsigned src_covered, - unsigned trg_covered, - double s2t_ratio) const { - if (src_covered == src.size() || trg_covered == trg.size()) { - assert(src_covered == src.size()); - assert(trg_covered == trg.size()); - return 0; - } - static const WordID kNULL = TD::Convert("<eps>"); - const prob_t uniform_alignment(1.0 / (src.size() - src_covered + 1)); - // TODO factor in expected length ratio - prob_t e; e.logeq(Md::log_poisson(trg.size() - trg_covered, (src.size() - src_covered) * s2t_ratio)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_covered; j < trg.size(); ++j) { - prob_t p = (*m1)(kNULL, trg[j]) + prob_t(1e-12); - for (unsigned i = src_covered; i < src.size(); ++i) - p += (*m1)(src[i], trg[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg[j]) << " | " << TD::GetString(src) << ") = 0!\n"; - assert(!"failed"); - } - p *= uniform_alignment; - e *= p; - } - // TODO factor in expected length ratio - const prob_t inv_uniform(1.0 / (trg.size() - trg_covered + 1.0)); - prob_t inv; - inv.logeq(Md::log_poisson(src.size() - src_covered, (trg.size() - trg_covered) / s2t_ratio)); - for (unsigned i = src_covered; i < src.size(); ++i) { - prob_t p = (*m1inv)(kNULL, src[i]) + prob_t(1e-12); - for (unsigned j = trg_covered; j < trg.size(); ++j) - p += (*m1inv)(trg[j], src[i]); - if (p.is_0()) { - cerr << "ERROR: p_inv(" << TD::Convert(src[i]) << " | " << TD::GetString(trg) << ") = 0!\n"; - assert(!"failed"); - } - p *= inv_uniform; - inv *= p; - } - return (log(e) + log(inv)) / 2; -} - -void BackwardEstimator::InitializeGrid(const vector<WordID>& src, - const vector<WordID>& trg, - const Reachability& r, - double s2t_ratio, - float* grid) const { - queue<pair<int,int> > q; - q.push(make_pair(0,0)); - Array2D<bool> done(src.size()+1, trg.size()+1, false); - //cerr << TD::GetString(src) << " ||| " << TD::GetString(trg) << endl; - while(!q.empty()) { - const pair<int,int> n = q.front(); - q.pop(); - if (done(n.first,n.second)) continue; - done(n.first,n.second) = true; - - float lp = ComputeBackwardProb(src, trg, n.first, n.second, s2t_ratio); - if (n.first == 0 && n.second == 0) grid[0] = lp; - //cerr << " " << n.first << "," << n.second << "\t" << lp << endl; - - if (n.first == src.size() || n.second == trg.size()) continue; - const vector<pair<short,short> >& edges = r.valid_deltas[n.first][n.second]; - for (int i = 0; i < edges.size(); ++i) - q.push(make_pair(n.first + edges[i].first, n.second + edges[i].second)); - } - //static int cc = 0; ++cc; if (cc == 80) exit(1); -} - diff --git a/gi/pf/backward.h b/gi/pf/backward.h deleted file mode 100644 index e67eff0c..00000000 --- a/gi/pf/backward.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef _BACKWARD_H_ -#define _BACKWARD_H_ - -#include <vector> -#include <string> -#include "wordid.h" - -struct Reachability; -struct Model1; - -struct BackwardEstimator { - BackwardEstimator(const std::string& s2t, - const std::string& t2s); - ~BackwardEstimator(); - - void InitializeGrid(const std::vector<WordID>& src, - const std::vector<WordID>& trg, - const Reachability& r, - double src2trg_ratio, - float* grid) const; - - private: - float ComputeBackwardProb(const std::vector<WordID>& src, - const std::vector<WordID>& trg, - unsigned src_covered, - unsigned trg_covered, - double src2trg_ratio) const; - - Model1* m1; - Model1* m1inv; -}; - -#endif diff --git a/gi/pf/base_distributions.cc b/gi/pf/base_distributions.cc deleted file mode 100644 index 57e0bbe1..00000000 --- a/gi/pf/base_distributions.cc +++ /dev/null @@ -1,241 +0,0 @@ -#include "base_distributions.h" - -#include <iostream> - -#include "filelib.h" - -using namespace std; - -TableLookupBase::TableLookupBase(const string& fname) { - cerr << "TableLookupBase reading from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - const WordID kDIV = TD::Convert("|||"); - vector<WordID> tmp; - vector<int> le, lf; - TRule x; - x.lhs_ = -TD::Convert("X"); - bool flag = false; - while(getline(in, line)) { - ++lc; - if (lc % 1000000 == 0) { cerr << " [" << lc << ']' << endl; flag = false; } - else if (lc % 25000 == 0) { cerr << '.' << flush; flag = true; } - tmp.clear(); - TD::ConvertSentence(line, &tmp); - x.f_.clear(); - x.e_.clear(); - size_t pos = 0; - int cc = 0; - while(pos < tmp.size()) { - const WordID cur = tmp[pos++]; - if (cur == kDIV) { - ++cc; - } else if (cc == 0) { - x.f_.push_back(cur); - } else if (cc == 1) { - x.e_.push_back(cur); - } else if (cc == 2) { - table[x].logeq(atof(TD::Convert(cur).c_str())); - ++cc; - } else { - if (flag) cerr << endl; - cerr << "Bad format in " << lc << ": " << line << endl; abort(); - } - } - if (cc != 3) { - if (flag) cerr << endl; - cerr << "Bad format in " << lc << ": " << line << endl; abort(); - } - } - if (flag) cerr << endl; - cerr << " read " << lc << " entries\n"; -} - -prob_t PhraseConditionalUninformativeUnigramBase::p0(const vector<WordID>& vsrc, - const vector<WordID>& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t p; - p.logeq(Md::log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - //p.logeq(log_poisson(elen, 1)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) - p *= u(vtrg[i + start_trg]); // draw e_i ~Uniform - return p; -} - -prob_t PhraseConditionalUninformativeBase::p0(const vector<WordID>& vsrc, - const vector<WordID>& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t p; - //p.logeq(log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - p.logeq(Md::log_poisson(elen, 1)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) - p *= kUNIFORM_TARGET; // draw e_i ~Uniform - return p; -} - -void Model1::LoadModel1(const string& fname) { - cerr << "Loading Model 1 parameters from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - while(getline(in, line)) { - ++lc; - int cur = 0; - int start = 0; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - const WordID src = TD::Convert(&line[0]); - ++cur; - start = cur; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - WordID trg = TD::Convert(&line[start]); - const double logprob = strtod(&line[cur + 1], NULL); - if (src >= ttable.size()) ttable.resize(src + 1); - ttable[src][trg].logeq(logprob); - } - cerr << " read " << lc << " parameters.\n"; -} - -prob_t PhraseConditionalBase::p0(const vector<WordID>& vsrc, - const vector<WordID>& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t p; - p.logeq(Md::log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) { // for each position i in e-RHS - const WordID trg = vtrg[i + start_trg]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? 0 : vsrc[j + start_src]; - tp += kM1MIXTURE * model1(src, trg); - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - if (p.is_0()) { - cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; - abort(); - } - return p; -} - -prob_t PhraseJointBase::p0(const vector<WordID>& vsrc, - const vector<WordID>& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t p; - p.logeq(Md::log_poisson(flen, 1.0)); // flen ~Pois(1) - // elen | flen ~Pois(flen + 0.01) - prob_t ptrglen; ptrglen.logeq(Md::log_poisson(elen, flen + 0.01)); - p *= ptrglen; - p *= kUNIFORM_SOURCE.pow(flen); // each f in F ~Uniform - for (int i = 0; i < elen; ++i) { // for each position i in E - const WordID trg = vtrg[i + start_trg]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? 0 : vsrc[j + start_src]; - tp += kM1MIXTURE * model1(src, trg); - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - if (p.is_0()) { - cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; - abort(); - } - return p; -} - -prob_t PhraseJointBase_BiDir::p0(const vector<WordID>& vsrc, - const vector<WordID>& vtrg, - int start_src, int start_trg) const { - const int flen = vsrc.size() - start_src; - const int elen = vtrg.size() - start_trg; - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t uniform_trg_alignment; uniform_trg_alignment.logeq(-log(elen + 1)); - - prob_t p1; - p1.logeq(Md::log_poisson(flen, 1.0)); // flen ~Pois(1) - // elen | flen ~Pois(flen + 0.01) - prob_t ptrglen; ptrglen.logeq(Md::log_poisson(elen, flen + 0.01)); - p1 *= ptrglen; - p1 *= kUNIFORM_SOURCE.pow(flen); // each f in F ~Uniform - for (int i = 0; i < elen; ++i) { // for each position i in E - const WordID trg = vtrg[i + start_trg]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? 0 : vsrc[j + start_src]; - tp += kM1MIXTURE * model1(src, trg); - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p1 *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - if (p1.is_0()) { - cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; - abort(); - } - - prob_t p2; - p2.logeq(Md::log_poisson(elen, 1.0)); // elen ~Pois(1) - // flen | elen ~Pois(flen + 0.01) - prob_t psrclen; psrclen.logeq(Md::log_poisson(flen, elen + 0.01)); - p2 *= psrclen; - p2 *= kUNIFORM_TARGET.pow(elen); // each f in F ~Uniform - for (int i = 0; i < flen; ++i) { // for each position i in E - const WordID src = vsrc[i + start_src]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < elen; ++j) { - const WordID trg = j < 0 ? 0 : vtrg[j + start_trg]; - tp += kM1MIXTURE * invmodel1(trg, src); - tp += kUNIFORM_MIXTURE * kUNIFORM_SOURCE; - } - tp *= uniform_trg_alignment; // draw a_i ~uniform - p2 *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - if (p2.is_0()) { - cerr << "Zero! " << vsrc << "\nTRG=" << vtrg << endl; - abort(); - } - - static const prob_t kHALF(0.5); - return (p1 + p2) * kHALF; -} - -JumpBase::JumpBase() : p(200) { - for (unsigned src_len = 1; src_len < 200; ++src_len) { - map<int, prob_t>& cpd = p[src_len]; - int min_jump = 1 - src_len; - int max_jump = src_len; - prob_t z; - for (int j = min_jump; j <= max_jump; ++j) { - prob_t& cp = cpd[j]; - if (j < 0) - cp.logeq(Md::log_poisson(1.5-j, 1)); - else if (j > 0) - cp.logeq(Md::log_poisson(j, 1)); - cp.poweq(0.2); - z += cp; - } - for (int j = min_jump; j <= max_jump; ++j) { - cpd[j] /= z; - } - } -} - diff --git a/gi/pf/base_distributions.h b/gi/pf/base_distributions.h deleted file mode 100644 index 41b513f8..00000000 --- a/gi/pf/base_distributions.h +++ /dev/null @@ -1,238 +0,0 @@ -#ifndef _BASE_MEASURES_H_ -#define _BASE_MEASURES_H_ - -#include <vector> -#include <map> -#include <string> -#include <cmath> -#include <iostream> -#include <cassert> - -#include "unigrams.h" -#include "trule.h" -#include "prob.h" -#include "tdict.h" -#include "sampler.h" -#include "m.h" -#include "os_phrase.h" - -struct Model1 { - explicit Model1(const std::string& fname) : - kNULL(TD::Convert("<eps>")), - kZERO() { - LoadModel1(fname); - } - - void LoadModel1(const std::string& fname); - - // returns prob 0 if src or trg is not found - const prob_t& operator()(WordID src, WordID trg) const { - if (src == 0) src = kNULL; - if (src < ttable.size()) { - const std::map<WordID, prob_t>& cpd = ttable[src]; - const std::map<WordID, prob_t>::const_iterator it = cpd.find(trg); - if (it != cpd.end()) - return it->second; - } - return kZERO; - } - - const WordID kNULL; - const prob_t kZERO; - std::vector<std::map<WordID, prob_t> > ttable; -}; - -struct PoissonUniformUninformativeBase { - explicit PoissonUniformUninformativeBase(const unsigned ves) : kUNIFORM(1.0 / ves) {} - prob_t operator()(const TRule& r) const { - prob_t p; p.logeq(Md::log_poisson(r.e_.size(), 1.0)); - prob_t q = kUNIFORM; q.poweq(r.e_.size()); - p *= q; - return p; - } - void Summary() const {} - void ResampleHyperparameters(MT19937*) {} - void Increment(const TRule&) {} - void Decrement(const TRule&) {} - prob_t Likelihood() const { return prob_t::One(); } - const prob_t kUNIFORM; -}; - -struct CompletelyUniformBase { - explicit CompletelyUniformBase(const unsigned ves) : kUNIFORM(1.0 / ves) {} - prob_t operator()(const TRule&) const { - return kUNIFORM; - } - void Summary() const {} - void ResampleHyperparameters(MT19937*) {} - void Increment(const TRule&) {} - void Decrement(const TRule&) {} - prob_t Likelihood() const { return prob_t::One(); } - const prob_t kUNIFORM; -}; - -struct UnigramWordBase { - explicit UnigramWordBase(const std::string& fname) : un(fname) {} - prob_t operator()(const TRule& r) const { - return un(r.e_); - } - const UnigramWordModel un; -}; - -struct RuleHasher { - size_t operator()(const TRule& r) const { - return hash_value(r); - } -}; - -struct TableLookupBase { - TableLookupBase(const std::string& fname); - - prob_t operator()(const TRule& rule) const { - const std::tr1::unordered_map<TRule,prob_t,RuleHasher>::const_iterator it = table.find(rule); - if (it == table.end()) { - std::cerr << rule << " not found\n"; - abort(); - } - return it->second; - } - - void ResampleHyperparameters(MT19937*) {} - void Increment(const TRule&) {} - void Decrement(const TRule&) {} - prob_t Likelihood() const { return prob_t::One(); } - void Summary() const {} - - std::tr1::unordered_map<TRule,prob_t,RuleHasher> table; -}; - -struct PhraseConditionalUninformativeBase { - explicit PhraseConditionalUninformativeBase(const unsigned vocab_e_size) : - kUNIFORM_TARGET(1.0 / vocab_e_size) { - assert(vocab_e_size > 0); - } - - // return p0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector<WordID>& vsrc, const std::vector<WordID>& vtrg, int start_src, int start_trg) const; - - void Summary() const {} - void ResampleHyperparameters(MT19937*) {} - void Increment(const TRule&) {} - void Decrement(const TRule&) {} - prob_t Likelihood() const { return prob_t::One(); } - const prob_t kUNIFORM_TARGET; -}; - -struct PhraseConditionalUninformativeUnigramBase { - explicit PhraseConditionalUninformativeUnigramBase(const std::string& file, const unsigned vocab_e_size) : u(file, vocab_e_size) {} - - // return p0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector<WordID>& vsrc, const std::vector<WordID>& vtrg, int start_src, int start_trg) const; - - const UnigramModel u; -}; - -struct PhraseConditionalBase { - explicit PhraseConditionalBase(const Model1& m1, const double m1mixture, const unsigned vocab_e_size) : - model1(m1), - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_TARGET(1.0 / vocab_e_size) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - } - - // return p0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector<WordID>& vsrc, const std::vector<WordID>& vtrg, int start_src, int start_trg) const; - - const Model1& model1; - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_TARGET; -}; - -struct PhraseJointBase { - explicit PhraseJointBase(const Model1& m1, const double m1mixture, const unsigned vocab_e_size, const unsigned vocab_f_size) : - model1(m1), - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_SOURCE(1.0 / vocab_f_size), - kUNIFORM_TARGET(1.0 / vocab_e_size) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - } - - // return p0 of rule.e_ , rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector<WordID>& vsrc, const std::vector<WordID>& vtrg, int start_src, int start_trg) const; - - const Model1& model1; - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_SOURCE; - const prob_t kUNIFORM_TARGET; -}; - -struct PhraseJointBase_BiDir { - explicit PhraseJointBase_BiDir(const Model1& m1, - const Model1& im1, - const double m1mixture, - const unsigned vocab_e_size, - const unsigned vocab_f_size) : - model1(m1), - invmodel1(im1), - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_SOURCE(1.0 / vocab_f_size), - kUNIFORM_TARGET(1.0 / vocab_e_size) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - } - - // return p0 of rule.e_ , rule.f_ - prob_t operator()(const TRule& rule) const { - return p0(rule.f_, rule.e_, 0, 0); - } - - prob_t p0(const std::vector<WordID>& vsrc, const std::vector<WordID>& vtrg, int start_src, int start_trg) const; - - const Model1& model1; - const Model1& invmodel1; - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_SOURCE; - const prob_t kUNIFORM_TARGET; -}; - -// base distribution for jump size multinomials -// basically p(0) = 0 and then, p(1) is max, and then -// you drop as you move to the max jump distance -struct JumpBase { - JumpBase(); - - const prob_t& operator()(int jump, unsigned src_len) const { - assert(jump != 0); - const std::map<int, prob_t>::const_iterator it = p[src_len].find(jump); - assert(it != p[src_len].end()); - return it->second; - } - std::vector<std::map<int, prob_t> > p; -}; - - -#endif diff --git a/gi/pf/bayes_lattice_score.cc b/gi/pf/bayes_lattice_score.cc deleted file mode 100644 index 70cb8dc2..00000000 --- a/gi/pf/bayes_lattice_score.cc +++ /dev/null @@ -1,309 +0,0 @@ -#include <iostream> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "inside_outside.h" -#include "hg.h" -#include "hg_io.h" -#include "bottom_up_parser.h" -#include "fdict.h" -#include "grammar.h" -#include "m.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr<MT19937> prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("input,i",po::value<string>(),"Read parallel data from") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -unsigned ReadCorpus(const string& filename, - vector<Lattice>* e, - set<WordID>* vocab_e) { - e->clear(); - vocab_e->clear(); - ReadFile rf(filename); - istream* in = rf.stream(); - assert(*in); - string line; - unsigned toks = 0; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(Lattice()); - Lattice& le = e->back(); - LatticeTools::ConvertTextOrPLF(line, & le); - for (unsigned i = 0; i < le.size(); ++i) - for (unsigned j = 0; j < le[i].size(); ++j) - vocab_e->insert(le[i][j].label); - toks += le.size(); - } - return toks; -} - -struct BaseModel { - explicit BaseModel(unsigned tc) : - unif(1.0 / tc), p(prob_t::One()) {} - prob_t prob(const TRule& r) const { - return unif; - } - void increment(const TRule& r, MT19937* rng) { - p *= prob(r); - } - void decrement(const TRule& r, MT19937* rng) { - p /= prob(r); - } - prob_t Likelihood() const { - return p; - } - const prob_t unif; - prob_t p; -}; - -struct UnigramModel { - explicit UnigramModel(unsigned tc) : base(tc), crp(1,1,1,1), glue(1,1,1,1) {} - BaseModel base; - CCRP<TRule> crp; - CCRP<TRule> glue; - - prob_t Prob(const TRule& r) const { - if (r.Arity() != 0) { - return glue.prob(r, prob_t(0.5)); - } - return crp.prob(r, base.prob(r)); - } - - int Increment(const TRule& r, MT19937* rng) { - if (r.Arity() != 0) { - glue.increment(r, 0.5, rng); - return 0; - } else { - if (crp.increment(r, base.prob(r), rng)) { - base.increment(r, rng); - return 1; - } - return 0; - } - } - - int Decrement(const TRule& r, MT19937* rng) { - if (r.Arity() != 0) { - glue.decrement(r, rng); - return 0; - } else { - if (crp.decrement(r, rng)) { - base.decrement(r, rng); - return -1; - } - return 0; - } - } - - prob_t Likelihood() const { - prob_t p; - p.logeq(crp.log_crp_prob() + glue.log_crp_prob()); - p *= base.Likelihood(); - return p; - } - - void ResampleHyperparameters(MT19937* rng) { - crp.resample_hyperparameters(rng); - glue.resample_hyperparameters(rng); - cerr << " d=" << crp.discount() << ", s=" << crp.strength() << "\t STOP d=" << glue.discount() << ", s=" << glue.strength() << endl; - } -}; - -UnigramModel* plm; - -void SampleDerivation(const Hypergraph& hg, MT19937* rng, vector<unsigned>* sampled_deriv) { - vector<prob_t> node_probs; - Inside<prob_t, EdgeProb>(hg, &node_probs); - queue<unsigned> q; - q.push(hg.nodes_.size() - 2); - while(!q.empty()) { - unsigned cur_node_id = q.front(); -// cerr << "NODE=" << cur_node_id << endl; - q.pop(); - const Hypergraph::Node& node = hg.nodes_[cur_node_id]; - const unsigned num_in_edges = node.in_edges_.size(); - unsigned sampled_edge = 0; - if (num_in_edges == 1) { - sampled_edge = node.in_edges_[0]; - } else { - //prob_t z; - assert(num_in_edges > 1); - SampleSet<prob_t> ss; - for (unsigned j = 0; j < num_in_edges; ++j) { - const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; - prob_t p = edge.edge_prob_; - for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) - p *= node_probs[edge.tail_nodes_[k]]; - ss.add(p); -// cerr << log(ss[j]) << " ||| " << edge.rule_->AsString() << endl; - //z += p; - } -// for (unsigned j = 0; j < num_in_edges; ++j) { -// const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; -// cerr << exp(log(ss[j] / z)) << " ||| " << edge.rule_->AsString() << endl; -// } -// cerr << " --- \n"; - sampled_edge = node.in_edges_[rng->SelectSample(ss)]; - } - sampled_deriv->push_back(sampled_edge); - const Hypergraph::Edge& edge = hg.edges_[sampled_edge]; - for (unsigned j = 0; j < edge.tail_nodes_.size(); ++j) { - q.push(edge.tail_nodes_[j]); - } - } -// for (unsigned i = 0; i < sampled_deriv->size(); ++i) { -// cerr << *hg.edges_[(*sampled_deriv)[i]].rule_ << endl; -// } -} - -void IncrementDerivation(const Hypergraph& hg, const vector<unsigned>& d, UnigramModel* plm, MT19937* rng) { - for (unsigned i = 0; i < d.size(); ++i) - plm->Increment(*hg.edges_[d[i]].rule_, rng); -} - -void DecrementDerivation(const Hypergraph& hg, const vector<unsigned>& d, UnigramModel* plm, MT19937* rng) { - for (unsigned i = 0; i < d.size(); ++i) - plm->Decrement(*hg.edges_[d[i]].rule_, rng); -} - -prob_t TotalProb(const Hypergraph& hg) { - return Inside<prob_t, EdgeProb>(hg); -} - -void IncrementLatticePath(const Hypergraph& hg, const vector<unsigned>& d, Lattice* pl) { - Lattice& lat = *pl; - for (int i = 0; i < d.size(); ++i) { - const Hypergraph::Edge& edge = hg.edges_[d[i]]; - if (edge.rule_->Arity() != 0) continue; - WordID sym = edge.rule_->e_[0]; - vector<LatticeArc>& las = lat[edge.i_]; - int dist = edge.j_ - edge.i_; - assert(dist > 0); - for (int j = 0; j < las.size(); ++j) { - if (las[j].dist2next == dist && - las[j].label == sym) { - las[j].cost += 1; - } - } - } -} - -int main(int argc, char** argv) { - po::variables_map conf; - - InitCommandLine(argc, argv, &conf); - vector<GrammarPtr> grammars(2); - grammars[0].reset(new GlueGrammar("S","X")); - const unsigned samples = conf["samples"].as<unsigned>(); - - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - vector<Lattice> corpuse; - set<WordID> vocabe; - cerr << "Reading corpus...\n"; - const unsigned toks = ReadCorpus(conf["input"].as<string>(), &corpuse, &vocabe); - cerr << "E-corpus size: " << corpuse.size() << " lattices\t (" << vocabe.size() << " word types)\n"; - UnigramModel lm(vocabe.size()); - vector<Hypergraph> hgs(corpuse.size()); - vector<vector<unsigned> > derivs(corpuse.size()); - for (int i = 0; i < corpuse.size(); ++i) { - grammars[1].reset(new PassThroughGrammar(corpuse[i], "X")); - ExhaustiveBottomUpParser parser("S", grammars); - bool res = parser.Parse(corpuse[i], &hgs[i]); // exhaustive parse - assert(res); - } - - double csamples = 0; - for (int SS=0; SS < samples; ++SS) { - const bool is_last = ((samples - 1) == SS); - prob_t dlh = prob_t::One(); - bool record_sample = (SS > (samples * 1 / 3) && (SS % 5 == 3)); - if (record_sample) csamples++; - for (int ci = 0; ci < corpuse.size(); ++ci) { - Lattice& lat = corpuse[ci]; - Hypergraph& hg = hgs[ci]; - vector<unsigned>& d = derivs[ci]; - if (!is_last) DecrementDerivation(hg, d, &lm, &rng); - for (unsigned i = 0; i < hg.edges_.size(); ++i) { - TRule& r = *hg.edges_[i].rule_; - if (r.Arity() != 0) - hg.edges_[i].edge_prob_ = prob_t::One(); - else - hg.edges_[i].edge_prob_ = lm.Prob(r); - } - if (!is_last) { - d.clear(); - SampleDerivation(hg, &rng, &d); - IncrementDerivation(hg, derivs[ci], &lm, &rng); - } else { - prob_t p = TotalProb(hg); - dlh *= p; - cerr << " p(sentence) = " << log(p) << "\t" << log(dlh) << endl; - } - if (record_sample) IncrementLatticePath(hg, derivs[ci], &lat); - } - double llh = log(lm.Likelihood()); - cerr << "LLH=" << llh << "\tENTROPY=" << (-llh / log(2) / toks) << "\tPPL=" << pow(2, -llh / log(2) / toks) << endl; - if (SS % 10 == 9) lm.ResampleHyperparameters(&rng); - if (is_last) { - double z = log(dlh); - cerr << "TOTAL_PROB=" << z << "\tENTROPY=" << (-z / log(2) / toks) << "\tPPL=" << pow(2, -z / log(2) / toks) << endl; - } - } - cerr << lm.crp << endl; - cerr << lm.glue << endl; - for (int i = 0; i < corpuse.size(); ++i) { - for (int j = 0; j < corpuse[i].size(); ++j) - for (int k = 0; k < corpuse[i][j].size(); ++k) { - corpuse[i][j][k].cost /= csamples; - corpuse[i][j][k].cost += 1e-3; - corpuse[i][j][k].cost = log(corpuse[i][j][k].cost); - } - cout << HypergraphIO::AsPLF(corpuse[i]) << endl; - } - return 0; -} - diff --git a/gi/pf/brat.cc b/gi/pf/brat.cc deleted file mode 100644 index 832f22cf..00000000 --- a/gi/pf/brat.cc +++ /dev/null @@ -1,543 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/multi_array.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "cfg_wfst_composer.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -static unsigned kMAX_SRC_PHRASE; -static unsigned kMAX_TRG_PHRASE; -struct FSTState; - -double log_poisson(unsigned x, const double& lambda) { - assert(lambda > 0.0); - return log(lambda) * x - lgamma(x + 1) - lambda; -} - -struct ConditionalBase { - explicit ConditionalBase(const double m1mixture, const unsigned vocab_e_size, const string& model1fname) : - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_TARGET(1.0 / vocab_e_size), - kNULL(TD::Convert("<eps>")) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - LoadModel1(model1fname); - } - - void LoadModel1(const string& fname) { - cerr << "Loading Model 1 parameters from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - while(getline(in, line)) { - ++lc; - int cur = 0; - int start = 0; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - const WordID src = TD::Convert(&line[0]); - ++cur; - start = cur; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - WordID trg = TD::Convert(&line[start]); - const double logprob = strtod(&line[cur + 1], NULL); - if (src >= ttable.size()) ttable.resize(src + 1); - ttable[src][trg].logeq(logprob); - } - cerr << " read " << lc << " parameters.\n"; - } - - // return logp0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - const int flen = rule.f_.size(); - const int elen = rule.e_.size(); - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t p; - p.logeq(log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) { // for each position i in e-RHS - const WordID trg = rule.e_[i]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? kNULL : rule.f_[j]; - const map<WordID, prob_t>::const_iterator it = ttable[src].find(trg); - if (it != ttable[src].end()) { - tp += kM1MIXTURE * it->second; - } - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - return p; - } - - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_TARGET; - const WordID kNULL; - vector<map<WordID, prob_t> > ttable; -}; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("input,i",po::value<string>(),"Read parallel data from") - ("max_src_phrase",po::value<unsigned>()->default_value(3),"Maximum length of source language phrases") - ("max_trg_phrase",po::value<unsigned>()->default_value(3),"Maximum length of target language phrases") - ("model1,m",po::value<string>(),"Model 1 parameters (used in base distribution)") - ("model1_interpolation_weight",po::value<double>()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector<vector<WordID> >* f, - vector<vector<int> >* e, - set<int>* vocab_f, - set<int>* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector<WordID> tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector<int>()); - f->push_back(vector<int>()); - vector<int>& le = e->back(); - vector<int>& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -struct UniphraseLM { - UniphraseLM(const vector<vector<int> >& corpus, - const set<int>& vocab, - const po::variables_map& conf) : - phrases_(1,1), - gen_(1,1), - corpus_(corpus), - uniform_word_(1.0 / vocab.size()), - gen_p0_(0.5), - p_end_(0.5), - use_poisson_(conf.count("poisson_length") > 0) {} - - void ResampleHyperparameters(MT19937* rng) { - phrases_.resample_hyperparameters(rng); - gen_.resample_hyperparameters(rng); - cerr << " " << phrases_.alpha(); - } - - CCRP_NoTable<vector<int> > phrases_; - CCRP_NoTable<bool> gen_; - vector<vector<bool> > z_; // z_[i] is there a phrase boundary after the ith word - const vector<vector<int> >& corpus_; - const double uniform_word_; - const double gen_p0_; - const double p_end_; // in base length distribution, p of the end of a phrase - const bool use_poisson_; -}; - -struct Reachability { - boost::multi_array<bool, 4> edges; // edges[src_covered][trg_covered][x][trg_delta] is this edge worth exploring? - boost::multi_array<short, 2> max_src_delta; // msd[src_covered][trg_covered] -- the largest src delta that's valid - - Reachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) : - edges(boost::extents[srclen][trglen][src_max_phrase_len+1][trg_max_phrase_len+1]), - max_src_delta(boost::extents[srclen][trglen]) { - ComputeReachability(srclen, trglen, src_max_phrase_len, trg_max_phrase_len); - } - - private: - struct SState { - SState() : prev_src_covered(), prev_trg_covered() {} - SState(int i, int j) : prev_src_covered(i), prev_trg_covered(j) {} - int prev_src_covered; - int prev_trg_covered; - }; - - struct NState { - NState() : next_src_covered(), next_trg_covered() {} - NState(int i, int j) : next_src_covered(i), next_trg_covered(j) {} - int next_src_covered; - int next_trg_covered; - }; - - void ComputeReachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) { - typedef boost::multi_array<vector<SState>, 2> array_type; - array_type a(boost::extents[srclen + 1][trglen + 1]); - a[0][0].push_back(SState()); - for (int i = 0; i < srclen; ++i) { - for (int j = 0; j < trglen; ++j) { - if (a[i][j].size() == 0) continue; - const SState prev(i,j); - for (int k = 1; k <= src_max_phrase_len; ++k) { - if ((i + k) > srclen) continue; - for (int l = 1; l <= trg_max_phrase_len; ++l) { - if ((j + l) > trglen) continue; - a[i + k][j + l].push_back(prev); - } - } - } - } - a[0][0].clear(); - cerr << "Final cell contains " << a[srclen][trglen].size() << " back pointers\n"; - assert(a[srclen][trglen].size() > 0); - - typedef boost::multi_array<bool, 2> rarray_type; - rarray_type r(boost::extents[srclen + 1][trglen + 1]); -// typedef boost::multi_array<vector<NState>, 2> narray_type; -// narray_type b(boost::extents[srclen + 1][trglen + 1]); - r[srclen][trglen] = true; - for (int i = srclen; i >= 0; --i) { - for (int j = trglen; j >= 0; --j) { - vector<SState>& prevs = a[i][j]; - if (!r[i][j]) { prevs.clear(); } -// const NState nstate(i,j); - for (int k = 0; k < prevs.size(); ++k) { - r[prevs[k].prev_src_covered][prevs[k].prev_trg_covered] = true; - int src_delta = i - prevs[k].prev_src_covered; - edges[prevs[k].prev_src_covered][prevs[k].prev_trg_covered][src_delta][j - prevs[k].prev_trg_covered] = true; - short &msd = max_src_delta[prevs[k].prev_src_covered][prevs[k].prev_trg_covered]; - if (src_delta > msd) msd = src_delta; -// b[prevs[k].prev_src_covered][prevs[k].prev_trg_covered].push_back(nstate); - } - } - } - assert(!edges[0][0][1][0]); - assert(!edges[0][0][0][1]); - assert(!edges[0][0][0][0]); - cerr << " MAX SRC DELTA[0][0] = " << max_src_delta[0][0] << endl; - assert(max_src_delta[0][0] > 0); - //cerr << "First cell contains " << b[0][0].size() << " forward pointers\n"; - //for (int i = 0; i < b[0][0].size(); ++i) { - // cerr << " -> (" << b[0][0][i].next_src_covered << "," << b[0][0][i].next_trg_covered << ")\n"; - //} - } -}; - -ostream& operator<<(ostream& os, const FSTState& q); -struct FSTState { - explicit FSTState(int src_size) : - trg_covered_(), - src_covered_(), - src_coverage_(src_size) {} - - FSTState(short trg_covered, short src_covered, const vector<bool>& src_coverage, const vector<short>& src_prefix) : - trg_covered_(trg_covered), - src_covered_(src_covered), - src_coverage_(src_coverage), - src_prefix_(src_prefix) { - if (src_coverage_.size() == src_covered) { - assert(src_prefix.size() == 0); - } - } - - // if we extend by the word at src_position, what are - // the next states that are reachable and lie on a valid - // path to the final state? - vector<FSTState> Extensions(int src_position, int src_len, int trg_len, const Reachability& r) const { - assert(src_position < src_coverage_.size()); - if (src_coverage_[src_position]) { - cerr << "Trying to extend " << *this << " with position " << src_position << endl; - abort(); - } - vector<bool> ncvg = src_coverage_; - ncvg[src_position] = true; - - vector<FSTState> res; - const int trg_remaining = trg_len - trg_covered_; - if (trg_remaining <= 0) { - cerr << "Target appears to have been covered: " << *this << " (trg_len=" << trg_len << ",trg_covered=" << trg_covered_ << ")" << endl; - abort(); - } - const int src_remaining = src_len - src_covered_; - if (src_remaining <= 0) { - cerr << "Source appears to have been covered: " << *this << endl; - abort(); - } - - for (int tc = 1; tc <= kMAX_TRG_PHRASE; ++tc) { - if (r.edges[src_covered_][trg_covered_][src_prefix_.size() + 1][tc]) { - int nc = src_prefix_.size() + 1 + src_covered_; - res.push_back(FSTState(trg_covered_ + tc, nc, ncvg, vector<short>())); - } - } - - if ((src_prefix_.size() + 1) < r.max_src_delta[src_covered_][trg_covered_]) { - vector<short> nsp = src_prefix_; - nsp.push_back(src_position); - res.push_back(FSTState(trg_covered_, src_covered_, ncvg, nsp)); - } - - if (res.size() == 0) { - cerr << *this << " can't be extended!\n"; - abort(); - } - return res; - } - - short trg_covered_, src_covered_; - vector<bool> src_coverage_; - vector<short> src_prefix_; -}; -bool operator<(const FSTState& q, const FSTState& r) { - if (q.trg_covered_ != r.trg_covered_) return q.trg_covered_ < r.trg_covered_; - if (q.src_covered_!= r.src_covered_) return q.src_covered_ < r.src_covered_; - if (q.src_coverage_ != r.src_coverage_) return q.src_coverage_ < r.src_coverage_; - return q.src_prefix_ < r.src_prefix_; -} - -ostream& operator<<(ostream& os, const FSTState& q) { - os << "[" << q.trg_covered_ << " : "; - for (int i = 0; i < q.src_coverage_.size(); ++i) - os << q.src_coverage_[i]; - os << " : <"; - for (int i = 0; i < q.src_prefix_.size(); ++i) { - if (i != 0) os << ' '; - os << q.src_prefix_[i]; - } - return os << ">]"; -} - -struct MyModel { - MyModel(ConditionalBase& rcp0) : rp0(rcp0) {} - typedef unordered_map<vector<WordID>, CCRP_NoTable<TRule>, boost::hash<vector<WordID> > > SrcToRuleCRPMap; - - void DecrementRule(const TRule& rule) { - SrcToRuleCRPMap::iterator it = rules.find(rule.f_); - assert(it != rules.end()); - it->second.decrement(rule); - if (it->second.num_customers() == 0) rules.erase(it); - } - - void IncrementRule(const TRule& rule) { - SrcToRuleCRPMap::iterator it = rules.find(rule.f_); - if (it == rules.end()) { - CCRP_NoTable<TRule> crp(1,1); - it = rules.insert(make_pair(rule.f_, crp)).first; - } - it->second.increment(rule); - } - - // conditioned on rule.f_ - prob_t RuleConditionalProbability(const TRule& rule) const { - const prob_t base = rp0(rule); - SrcToRuleCRPMap::const_iterator it = rules.find(rule.f_); - if (it == rules.end()) { - return base; - } else { - const double lp = it->second.logprob(rule, log(base)); - prob_t q; q.logeq(lp); - return q; - } - } - - const ConditionalBase& rp0; - SrcToRuleCRPMap rules; -}; - -struct MyFST : public WFST { - MyFST(const vector<WordID>& ssrc, const vector<WordID>& strg, MyModel* m) : - src(ssrc), trg(strg), - r(src.size(),trg.size(),kMAX_SRC_PHRASE, kMAX_TRG_PHRASE), - model(m) { - FSTState in(src.size()); - cerr << " INIT: " << in << endl; - init = GetNode(in); - for (int i = 0; i < in.src_coverage_.size(); ++i) in.src_coverage_[i] = true; - in.src_covered_ = src.size(); - in.trg_covered_ = trg.size(); - cerr << "FINAL: " << in << endl; - final = GetNode(in); - } - virtual const WFSTNode* Final() const; - virtual const WFSTNode* Initial() const; - - const WFSTNode* GetNode(const FSTState& q); - map<FSTState, boost::shared_ptr<WFSTNode> > m; - const vector<WordID>& src; - const vector<WordID>& trg; - Reachability r; - const WFSTNode* init; - const WFSTNode* final; - MyModel* model; -}; - -struct MyNode : public WFSTNode { - MyNode(const FSTState& q, MyFST* fst) : state(q), container(fst) {} - virtual vector<pair<const WFSTNode*, TRulePtr> > ExtendInput(unsigned srcindex) const; - const FSTState state; - mutable MyFST* container; -}; - -vector<pair<const WFSTNode*, TRulePtr> > MyNode::ExtendInput(unsigned srcindex) const { - cerr << "EXTEND " << state << " with " << srcindex << endl; - vector<FSTState> ext = state.Extensions(srcindex, container->src.size(), container->trg.size(), container->r); - vector<pair<const WFSTNode*,TRulePtr> > res(ext.size()); - for (unsigned i = 0; i < ext.size(); ++i) { - res[i].first = container->GetNode(ext[i]); - if (ext[i].src_prefix_.size() == 0) { - const unsigned trg_from = state.trg_covered_; - const unsigned trg_to = ext[i].trg_covered_; - const unsigned prev_prfx_size = state.src_prefix_.size(); - res[i].second.reset(new TRule); - res[i].second->lhs_ = -TD::Convert("X"); - vector<WordID>& src = res[i].second->f_; - vector<WordID>& trg = res[i].second->e_; - src.resize(prev_prfx_size + 1); - for (unsigned j = 0; j < prev_prfx_size; ++j) - src[j] = container->src[state.src_prefix_[j]]; - src[prev_prfx_size] = container->src[srcindex]; - for (unsigned j = trg_from; j < trg_to; ++j) - trg.push_back(container->trg[j]); - res[i].second->scores_.set_value(FD::Convert("Proposal"), log(container->model->RuleConditionalProbability(*res[i].second))); - } - } - return res; -} - -const WFSTNode* MyFST::GetNode(const FSTState& q) { - boost::shared_ptr<WFSTNode>& res = m[q]; - if (!res) { - res.reset(new MyNode(q, this)); - } - return &*res; -} - -const WFSTNode* MyFST::Final() const { - return final; -} - -const WFSTNode* MyFST::Initial() const { - return init; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - kMAX_TRG_PHRASE = conf["max_trg_phrase"].as<unsigned>(); - kMAX_SRC_PHRASE = conf["max_src_phrase"].as<unsigned>(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - boost::shared_ptr<MT19937> prng; - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector<vector<int> > corpuse, corpusf; - set<int> vocabe, vocabf; - ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - ConditionalBase lp0(conf["model1_interpolation_weight"].as<double>(), - vocabe.size(), - conf["model1"].as<string>()); - MyModel m(lp0); - - TRule x("[X] ||| kAnwntR myN ||| at the convent ||| 0"); - m.IncrementRule(x); - TRule y("[X] ||| nY dyN ||| gave ||| 0"); - m.IncrementRule(y); - - - MyFST fst(corpusf[0], corpuse[0], &m); - ifstream in("./kimura.g"); - assert(in); - CFG_WFSTComposer comp(fst); - Hypergraph hg; - bool succeed = comp.Compose(&in, &hg); - hg.PrintGraphviz(); - if (succeed) { cerr << "SUCCESS.\n"; } else { cerr << "FAILURE REPORTED.\n"; } - -#if 0 - ifstream in2("./amnabooks.g"); - assert(in2); - MyFST fst2(corpusf[1], corpuse[1], &m); - CFG_WFSTComposer comp2(fst2); - Hypergraph hg2; - bool succeed2 = comp2.Compose(&in2, &hg2); - if (succeed2) { cerr << "SUCCESS.\n"; } else { cerr << "FAILURE REPORTED.\n"; } -#endif - - SparseVector<double> w; w.set_value(FD::Convert("Proposal"), 1.0); - hg.Reweight(w); - cerr << ViterbiFTree(hg) << endl; - return 0; -} - diff --git a/gi/pf/cbgi.cc b/gi/pf/cbgi.cc deleted file mode 100644 index 97f1ba34..00000000 --- a/gi/pf/cbgi.cc +++ /dev/null @@ -1,330 +0,0 @@ -#include <queue> -#include <sstream> -#include <iostream> - -#include <boost/unordered_map.hpp> -#include <boost/functional/hash.hpp> - -#include "sampler.h" -#include "filelib.h" -#include "hg_io.h" -#include "hg.h" -#include "ccrp_nt.h" -#include "trule.h" -#include "inside_outside.h" - -using namespace std; -using namespace std::tr1; - -double log_poisson(unsigned x, const double& lambda) { - assert(lambda > 0.0); - return log(lambda) * x - lgamma(x + 1) - lambda; -} - -double log_decay(unsigned x, const double& b) { - assert(b > 1.0); - assert(x > 0); - return log(b - 1) - x * log(b); -} - -struct SimpleBase { - SimpleBase(unsigned esize, unsigned fsize, unsigned ntsize = 144) : - uniform_e(-log(esize)), - uniform_f(-log(fsize)), - uniform_nt(-log(ntsize)) { - } - - // binomial coefficient - static double choose(unsigned n, unsigned k) { - return exp(lgamma(n + 1) - lgamma(k + 1) - lgamma(n - k + 1)); - } - - // count the number of patterns of terminals and NTs in the rule, given elen and flen - static double log_number_of_patterns(const unsigned flen, const unsigned elen) { - static vector<vector<double> > counts; - if (elen >= counts.size()) counts.resize(elen + 1); - if (flen >= counts[elen].size()) counts[elen].resize(flen + 1); - double& count = counts[elen][flen]; - if (count) return log(count); - const unsigned max_arity = min(elen, flen); - for (unsigned a = 0; a <= max_arity; ++a) - count += choose(elen, a) * choose(flen, a); - return log(count); - } - - // return logp0 of rule | LHS - double operator()(const TRule& rule) const { - const unsigned flen = rule.f_.size(); - const unsigned elen = rule.e_.size(); -#if 0 - double p = 0; - p += log_poisson(flen, 0.5); // flen ~Pois(0.5) - p += log_poisson(elen, flen); // elen | flen ~Pois(flen) - p -= log_number_of_patterns(flen, elen); // pattern | flen,elen ~Uniform - for (unsigned i = 0; i < flen; ++i) { // for each position in f-RHS - if (rule.f_[i] <= 0) // according to pattern - p += uniform_nt; // draw NT ~Uniform - else - p += uniform_f; // draw f terminal ~Uniform - } - p -= lgamma(rule.Arity() + 1); // draw permutation ~Uniform - for (unsigned i = 0; i < elen; ++i) { // for each position in e-RHS - if (rule.e_[i] > 0) // according to pattern - p += uniform_e; // draw e|f term ~Uniform - // TODO this should prob be model 1 - } -#else - double p = 0; - bool is_abstract = rule.f_[0] <= 0; - p += log(0.5); - if (is_abstract) { - if (flen == 2) p += log(0.99); else p += log(0.01); - } else { - p += log_decay(flen, 3); - } - - for (unsigned i = 0; i < flen; ++i) { // for each position in f-RHS - if (rule.f_[i] <= 0) // according to pattern - p += uniform_nt; // draw NT ~Uniform - else - p += uniform_f; // draw f terminal ~Uniform - } -#endif - return p; - } - const double uniform_e; - const double uniform_f; - const double uniform_nt; - vector<double> arities; -}; - -MT19937* rng = NULL; - -template <typename Base> -struct MHSamplerEdgeProb { - MHSamplerEdgeProb(const Hypergraph& hg, - const map<int, CCRP_NoTable<TRule> >& rdp, - const Base& logp0, - const bool exclude_multiword_terminals) : edge_probs(hg.edges_.size()) { - for (int i = 0; i < edge_probs.size(); ++i) { - const TRule& rule = *hg.edges_[i].rule_; - const map<int, CCRP_NoTable<TRule> >::const_iterator it = rdp.find(rule.lhs_); - assert(it != rdp.end()); - const CCRP_NoTable<TRule>& crp = it->second; - edge_probs[i].logeq(crp.logprob(rule, logp0(rule))); - if (exclude_multiword_terminals && rule.f_[0] > 0 && rule.f_.size() > 1) - edge_probs[i] = prob_t::Zero(); - } - } - inline prob_t operator()(const Hypergraph::Edge& e) const { - return edge_probs[e.id_]; - } - prob_t DerivationProb(const vector<int>& d) const { - prob_t p = prob_t::One(); - for (unsigned i = 0; i < d.size(); ++i) - p *= edge_probs[d[i]]; - return p; - } - vector<prob_t> edge_probs; -}; - -template <typename Base> -struct ModelAndData { - ModelAndData() : - base_lh(prob_t::One()), - logp0(10000, 10000), - mh_samples(), - mh_rejects() {} - - void SampleCorpus(const string& hgpath, int i); - void ResampleHyperparameters() { - for (map<int, CCRP_NoTable<TRule> >::iterator it = rules.begin(); it != rules.end(); ++it) - it->second.resample_hyperparameters(rng); - } - - CCRP_NoTable<TRule>& RuleCRP(int lhs) { - map<int, CCRP_NoTable<TRule> >::iterator it = rules.find(lhs); - if (it == rules.end()) { - rules.insert(make_pair(lhs, CCRP_NoTable<TRule>(1,1))); - it = rules.find(lhs); - } - return it->second; - } - - void IncrementRule(const TRule& rule) { - CCRP_NoTable<TRule>& crp = RuleCRP(rule.lhs_); - if (crp.increment(rule)) { - prob_t p; p.logeq(logp0(rule)); - base_lh *= p; - } - } - - void DecrementRule(const TRule& rule) { - CCRP_NoTable<TRule>& crp = RuleCRP(rule.lhs_); - if (crp.decrement(rule)) { - prob_t p; p.logeq(logp0(rule)); - base_lh /= p; - } - } - - void DecrementDerivation(const Hypergraph& hg, const vector<int>& d) { - for (unsigned i = 0; i < d.size(); ++i) { - const TRule& rule = *hg.edges_[d[i]].rule_; - DecrementRule(rule); - } - } - - void IncrementDerivation(const Hypergraph& hg, const vector<int>& d) { - for (unsigned i = 0; i < d.size(); ++i) { - const TRule& rule = *hg.edges_[d[i]].rule_; - IncrementRule(rule); - } - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (map<int, CCRP_NoTable<TRule> >::const_iterator it = rules.begin(); it != rules.end(); ++it) { - prob_t q; q.logeq(it->second.log_crp_prob()); - p *= q; - } - p *= base_lh; - return p; - } - - void ResampleDerivation(const Hypergraph& hg, vector<int>* sampled_derivation); - - map<int, CCRP_NoTable<TRule> > rules; // [lhs] -> distribution over RHSs - prob_t base_lh; - SimpleBase logp0; - vector<vector<int> > samples; // sampled derivations - unsigned int mh_samples; - unsigned int mh_rejects; -}; - -template <typename Base> -void ModelAndData<Base>::SampleCorpus(const string& hgpath, int n) { - vector<Hypergraph> hgs(n); hgs.clear(); - boost::unordered_map<TRule, unsigned> acc; - map<int, unsigned> tot; - for (int i = 0; i < n; ++i) { - ostringstream os; - os << hgpath << '/' << i << ".json.gz"; - if (!FileExists(os.str())) continue; - hgs.push_back(Hypergraph()); - ReadFile rf(os.str()); - HypergraphIO::ReadFromJSON(rf.stream(), &hgs.back()); - } - cerr << "Read " << hgs.size() << " alignment hypergraphs.\n"; - samples.resize(hgs.size()); - const unsigned SAMPLES = 2000; - const unsigned burnin = 3 * SAMPLES / 4; - const unsigned every = 20; - for (unsigned s = 0; s < SAMPLES; ++s) { - if (s % 10 == 0) { - if (s > 0) { cerr << endl; ResampleHyperparameters(); } - cerr << "[" << s << " LLH=" << log(Likelihood()) << " REJECTS=" << ((double)mh_rejects / mh_samples) << " LHS's=" << rules.size() << " base=" << log(base_lh) << "] "; - } - cerr << '.'; - for (unsigned i = 0; i < hgs.size(); ++i) { - ResampleDerivation(hgs[i], &samples[i]); - if (s > burnin && s % every == 0) { - for (unsigned j = 0; j < samples[i].size(); ++j) { - const TRule& rule = *hgs[i].edges_[samples[i][j]].rule_; - ++acc[rule]; - ++tot[rule.lhs_]; - } - } - } - } - cerr << endl; - for (boost::unordered_map<TRule,unsigned>::iterator it = acc.begin(); it != acc.end(); ++it) { - cout << it->first << " MyProb=" << log(it->second)-log(tot[it->first.lhs_]) << endl; - } -} - -template <typename Base> -void ModelAndData<Base>::ResampleDerivation(const Hypergraph& hg, vector<int>* sampled_deriv) { - vector<int> cur; - cur.swap(*sampled_deriv); - - const prob_t p_cur = Likelihood(); - DecrementDerivation(hg, cur); - if (cur.empty()) { - // first iteration, create restaurants - for (int i = 0; i < hg.edges_.size(); ++i) - RuleCRP(hg.edges_[i].rule_->lhs_); - } - MHSamplerEdgeProb<SimpleBase> wf(hg, rules, logp0, cur.empty()); -// MHSamplerEdgeProb<SimpleBase> wf(hg, rules, logp0, false); - const prob_t q_cur = wf.DerivationProb(cur); - vector<prob_t> node_probs; - Inside<prob_t, MHSamplerEdgeProb<SimpleBase> >(hg, &node_probs, wf); - queue<unsigned> q; - q.push(hg.nodes_.size() - 3); - while(!q.empty()) { - unsigned cur_node_id = q.front(); -// cerr << "NODE=" << cur_node_id << endl; - q.pop(); - const Hypergraph::Node& node = hg.nodes_[cur_node_id]; - const unsigned num_in_edges = node.in_edges_.size(); - unsigned sampled_edge = 0; - if (num_in_edges == 1) { - sampled_edge = node.in_edges_[0]; - } else { - prob_t z; - assert(num_in_edges > 1); - SampleSet<prob_t> ss; - for (unsigned j = 0; j < num_in_edges; ++j) { - const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; - prob_t p = wf.edge_probs[edge.id_]; // edge proposal prob - for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) - p *= node_probs[edge.tail_nodes_[k]]; - ss.add(p); -// cerr << log(ss[j]) << " ||| " << edge.rule_->AsString() << endl; - z += p; - } -// for (unsigned j = 0; j < num_in_edges; ++j) { -// const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; -// cerr << exp(log(ss[j] / z)) << " ||| " << edge.rule_->AsString() << endl; -// } -// cerr << " --- \n"; - sampled_edge = node.in_edges_[rng->SelectSample(ss)]; - } - sampled_deriv->push_back(sampled_edge); - const Hypergraph::Edge& edge = hg.edges_[sampled_edge]; - for (unsigned j = 0; j < edge.tail_nodes_.size(); ++j) { - q.push(edge.tail_nodes_[j]); - } - } - IncrementDerivation(hg, *sampled_deriv); - -// cerr << "sampled derivation contains " << sampled_deriv->size() << " edges\n"; -// cerr << "DERIV:\n"; -// for (int i = 0; i < sampled_deriv->size(); ++i) { -// cerr << " " << hg.edges_[(*sampled_deriv)[i]].rule_->AsString() << endl; -// } - - if (cur.empty()) return; // accept first sample - - ++mh_samples; - // only need to do MH if proposal is different to current state - if (cur != *sampled_deriv) { - const prob_t q_prop = wf.DerivationProb(*sampled_deriv); - const prob_t p_prop = Likelihood(); - if (!rng->AcceptMetropolisHastings(p_prop, p_cur, q_prop, q_cur)) { - ++mh_rejects; - DecrementDerivation(hg, *sampled_deriv); - IncrementDerivation(hg, cur); - swap(cur, *sampled_deriv); - } - } -} - -int main(int argc, char** argv) { - rng = new MT19937; - ModelAndData<SimpleBase> m; - m.SampleCorpus("./hgs", 50); - // m.SampleCorpus("./btec/hgs", 5000); - return 0; -} - diff --git a/gi/pf/cfg_wfst_composer.cc b/gi/pf/cfg_wfst_composer.cc deleted file mode 100644 index 21d5ec5b..00000000 --- a/gi/pf/cfg_wfst_composer.cc +++ /dev/null @@ -1,731 +0,0 @@ -#include "cfg_wfst_composer.h" - -#include <iostream> -#include <fstream> -#include <map> -#include <queue> -#include <tr1/unordered_map> -#include <tr1/unordered_set> - -#include <boost/shared_ptr.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> -#include "fast_lexical_cast.hpp" - -#include "phrasetable_fst.h" -#include "sparse_vector.h" -#include "tdict.h" -#include "hg.h" -#include "hg_remove_eps.h" - -namespace po = boost::program_options; -using namespace std; -using namespace std::tr1; - -WFSTNode::~WFSTNode() {} -WFST::~WFST() {} - -// Define the following macro if you want to see lots of debugging output -// when you run the chart parser -#undef DEBUG_CHART_PARSER - -// A few constants used by the chart parser /////////////// -static const int kMAX_NODES = 2000000; -static const string kPHRASE_STRING = "X"; -static bool constants_need_init = true; -static WordID kUNIQUE_START; -static WordID kPHRASE; -static TRulePtr kX1X2; -static TRulePtr kX1; -static WordID kEPS; -static TRulePtr kEPSRule; - -static void InitializeConstants() { - if (constants_need_init) { - kPHRASE = TD::Convert(kPHRASE_STRING) * -1; - kUNIQUE_START = TD::Convert("S") * -1; - kX1X2.reset(new TRule("[X] ||| [X,1] [X,2] ||| [X,1] [X,2]")); - kX1.reset(new TRule("[X] ||| [X,1] ||| [X,1]")); - kEPSRule.reset(new TRule("[X] ||| <eps> ||| <eps>")); - kEPS = TD::Convert("<eps>"); - constants_need_init = false; - } -} -//////////////////////////////////////////////////////////// - -class EGrammarNode { - friend bool CFG_WFSTComposer::Compose(const Hypergraph& src_forest, Hypergraph* trg_forest); - friend void AddGrammarRule(const string& r, map<WordID, EGrammarNode>* g); - public: -#ifdef DEBUG_CHART_PARSER - string hint; -#endif - EGrammarNode() : is_some_rule_complete(false), is_root(false) {} - const map<WordID, EGrammarNode>& GetTerminals() const { return tptr; } - const map<WordID, EGrammarNode>& GetNonTerminals() const { return ntptr; } - bool HasNonTerminals() const { return (!ntptr.empty()); } - bool HasTerminals() const { return (!tptr.empty()); } - bool RuleCompletes() const { - return (is_some_rule_complete || (ntptr.empty() && tptr.empty())); - } - bool GrammarContinues() const { - return !(ntptr.empty() && tptr.empty()); - } - bool IsRoot() const { - return is_root; - } - // these are the features associated with the rule from the start - // node up to this point. If you use these features, you must - // not Extend() this rule. - const SparseVector<double>& GetCFGProductionFeatures() const { - return input_features; - } - - const EGrammarNode* Extend(const WordID& t) const { - if (t < 0) { - map<WordID, EGrammarNode>::const_iterator it = ntptr.find(t); - if (it == ntptr.end()) return NULL; - return &it->second; - } else { - map<WordID, EGrammarNode>::const_iterator it = tptr.find(t); - if (it == tptr.end()) return NULL; - return &it->second; - } - } - - private: - map<WordID, EGrammarNode> tptr; - map<WordID, EGrammarNode> ntptr; - SparseVector<double> input_features; - bool is_some_rule_complete; - bool is_root; -}; -typedef map<WordID, EGrammarNode> EGrammar; // indexed by the rule LHS - -// edges are immutable once created -struct Edge { -#ifdef DEBUG_CHART_PARSER - static int id_count; - const int id; -#endif - const WordID cat; // lhs side of rule proved/being proved - const EGrammarNode* const dot; // dot position - const WFSTNode* const q; // start of span - const WFSTNode* const r; // end of span - const Edge* const active_parent; // back pointer, NULL for PREDICT items - const Edge* const passive_parent; // back pointer, NULL for SCAN and PREDICT items - TRulePtr tps; // translations - boost::shared_ptr<SparseVector<double> > features; // features from CFG rule - - bool IsPassive() const { - // when a rule is completed, this value will be set - return static_cast<bool>(features); - } - bool IsActive() const { return !IsPassive(); } - bool IsInitial() const { - return !(active_parent || passive_parent); - } - bool IsCreatedByScan() const { - return active_parent && !passive_parent && !dot->IsRoot(); - } - bool IsCreatedByPredict() const { - return dot->IsRoot(); - } - bool IsCreatedByComplete() const { - return active_parent && passive_parent; - } - - // constructor for PREDICT - Edge(WordID c, const EGrammarNode* d, const WFSTNode* q_and_r) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(q_and_r), r(q_and_r), active_parent(NULL), passive_parent(NULL), tps() {} - Edge(WordID c, const EGrammarNode* d, const WFSTNode* q_and_r, const Edge* act_parent) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(q_and_r), r(q_and_r), active_parent(act_parent), passive_parent(NULL), tps() {} - - // constructors for SCAN - Edge(WordID c, const EGrammarNode* d, const WFSTNode* i, const WFSTNode* j, - const Edge* act_par, const TRulePtr& translations) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(NULL), tps(translations) {} - - Edge(WordID c, const EGrammarNode* d, const WFSTNode* i, const WFSTNode* j, - const Edge* act_par, const TRulePtr& translations, - const SparseVector<double>& feats) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(NULL), tps(translations), - features(new SparseVector<double>(feats)) {} - - // constructors for COMPLETE - Edge(WordID c, const EGrammarNode* d, const WFSTNode* i, const WFSTNode* j, - const Edge* act_par, const Edge *pas_par) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(pas_par), tps() { - assert(pas_par->IsPassive()); - assert(act_par->IsActive()); - } - - Edge(WordID c, const EGrammarNode* d, const WFSTNode* i, const WFSTNode* j, - const Edge* act_par, const Edge *pas_par, const SparseVector<double>& feats) : -#ifdef DEBUG_CHART_PARSER - id(++id_count), -#endif - cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(pas_par), tps(), - features(new SparseVector<double>(feats)) { - assert(pas_par->IsPassive()); - assert(act_par->IsActive()); - } - - // constructor for COMPLETE query - Edge(const WFSTNode* _r) : -#ifdef DEBUG_CHART_PARSER - id(0), -#endif - cat(0), dot(NULL), q(NULL), - r(_r), active_parent(NULL), passive_parent(NULL), tps() {} - // constructor for MERGE quere - Edge(const WFSTNode* _q, int) : -#ifdef DEBUG_CHART_PARSER - id(0), -#endif - cat(0), dot(NULL), q(_q), - r(NULL), active_parent(NULL), passive_parent(NULL), tps() {} -}; -#ifdef DEBUG_CHART_PARSER -int Edge::id_count = 0; -#endif - -ostream& operator<<(ostream& os, const Edge& e) { - string type = "PREDICT"; - if (e.IsCreatedByScan()) - type = "SCAN"; - else if (e.IsCreatedByComplete()) - type = "COMPLETE"; - os << "[" -#ifdef DEBUG_CHART_PARSER - << '(' << e.id << ") " -#else - << '(' << &e << ") " -#endif - << "q=" << e.q << ", r=" << e.r - << ", cat="<< TD::Convert(e.cat*-1) << ", dot=" - << e.dot -#ifdef DEBUG_CHART_PARSER - << e.dot->hint -#endif - << (e.IsActive() ? ", Active" : ", Passive") - << ", " << type; -#ifdef DEBUG_CHART_PARSER - if (e.active_parent) { os << ", act.parent=(" << e.active_parent->id << ')'; } - if (e.passive_parent) { os << ", psv.parent=(" << e.passive_parent->id << ')'; } -#endif - if (e.tps) { os << ", tps=" << e.tps->AsString(); } - return os << ']'; -} - -struct Traversal { - const Edge* const edge; // result from the active / passive combination - const Edge* const active; - const Edge* const passive; - Traversal(const Edge* me, const Edge* a, const Edge* p) : edge(me), active(a), passive(p) {} -}; - -struct UniqueTraversalHash { - size_t operator()(const Traversal* t) const { - size_t x = 5381; - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(t->active); - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(t->passive); - x = ((x << 5) + x) ^ t->edge->IsActive(); - return x; - } -}; - -struct UniqueTraversalEquals { - size_t operator()(const Traversal* a, const Traversal* b) const { - return (a->passive == b->passive && a->active == b->active && a->edge->IsActive() == b->edge->IsActive()); - } -}; - -struct UniqueEdgeHash { - size_t operator()(const Edge* e) const { - size_t x = 5381; - if (e->IsActive()) { - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->dot); - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->q); - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->r); - x = ((x << 5) + x) ^ static_cast<size_t>(e->cat); - x += 13; - } else { // with passive edges, we don't care about the dot - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->q); - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->r); - x = ((x << 5) + x) ^ static_cast<size_t>(e->cat); - } - return x; - } -}; - -struct UniqueEdgeEquals { - bool operator()(const Edge* a, const Edge* b) const { - if (a->IsActive() != b->IsActive()) return false; - if (a->IsActive()) { - return (a->cat == b->cat) && (a->dot == b->dot) && (a->q == b->q) && (a->r == b->r); - } else { - return (a->cat == b->cat) && (a->q == b->q) && (a->r == b->r); - } - } -}; - -struct REdgeHash { - size_t operator()(const Edge* e) const { - size_t x = 5381; - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->r); - return x; - } -}; - -struct REdgeEquals { - bool operator()(const Edge* a, const Edge* b) const { - return (a->r == b->r); - } -}; - -struct QEdgeHash { - size_t operator()(const Edge* e) const { - size_t x = 5381; - x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->q); - return x; - } -}; - -struct QEdgeEquals { - bool operator()(const Edge* a, const Edge* b) const { - return (a->q == b->q); - } -}; - -struct EdgeQueue { - queue<const Edge*> q; - EdgeQueue() {} - void clear() { while(!q.empty()) q.pop(); } - bool HasWork() const { return !q.empty(); } - const Edge* Next() { const Edge* res = q.front(); q.pop(); return res; } - void AddEdge(const Edge* s) { q.push(s); } -}; - -class CFG_WFSTComposerImpl { - public: - CFG_WFSTComposerImpl(WordID start_cat, - const WFSTNode* q_0, - const WFSTNode* q_final) : start_cat_(start_cat), q_0_(q_0), q_final_(q_final) {} - - // returns false if the intersection is empty - bool Compose(const EGrammar& g, Hypergraph* forest) { - goal_node = NULL; - EGrammar::const_iterator sit = g.find(start_cat_); - forest->ReserveNodes(kMAX_NODES); - assert(sit != g.end()); - Edge* init = new Edge(start_cat_, &sit->second, q_0_); - assert(IncorporateNewEdge(init)); - while (exp_agenda.HasWork() || agenda.HasWork()) { - while(exp_agenda.HasWork()) { - const Edge* edge = exp_agenda.Next(); - FinishEdge(edge, forest); - } - if (agenda.HasWork()) { - const Edge* edge = agenda.Next(); -#ifdef DEBUG_CHART_PARSER - cerr << "processing (" << edge->id << ')' << endl; -#endif - if (edge->IsActive()) { - if (edge->dot->HasTerminals()) - DoScan(edge); - if (edge->dot->HasNonTerminals()) { - DoMergeWithPassives(edge); - DoPredict(edge, g); - } - } else { - DoComplete(edge); - } - } - } - if (goal_node) { - forest->PruneUnreachable(goal_node->id_); - RemoveEpsilons(forest, kEPS); - } - FreeAll(); - return goal_node; - } - - void FreeAll() { - for (int i = 0; i < free_list_.size(); ++i) - delete free_list_[i]; - free_list_.clear(); - for (int i = 0; i < traversal_free_list_.size(); ++i) - delete traversal_free_list_[i]; - traversal_free_list_.clear(); - all_traversals.clear(); - exp_agenda.clear(); - agenda.clear(); - tps2node.clear(); - edge2node.clear(); - all_edges.clear(); - passive_edges.clear(); - active_edges.clear(); - } - - ~CFG_WFSTComposerImpl() { - FreeAll(); - } - - // returns the total number of edges created during composition - int EdgesCreated() const { - return free_list_.size(); - } - - private: - void DoScan(const Edge* edge) { - // here, we assume that the FST will potentially have many more outgoing - // edges than the grammar, which will be just a couple. If you want to - // efficiently handle the case where both are relatively large, this code - // will need to change how the intersection is done. The best general - // solution would probably be the Baeza-Yates double binary search. - - const EGrammarNode* dot = edge->dot; - const WFSTNode* r = edge->r; - const map<WordID, EGrammarNode>& terms = dot->GetTerminals(); - for (map<WordID, EGrammarNode>::const_iterator git = terms.begin(); - git != terms.end(); ++git) { - - if (!(TD::Convert(git->first)[0] >= '0' && TD::Convert(git->first)[0] <= '9')) { - std::cerr << "TERMINAL SYMBOL: " << TD::Convert(git->first) << endl; - abort(); - } - std::vector<std::pair<const WFSTNode*, TRulePtr> > extensions = r->ExtendInput(atoi(TD::Convert(git->first).c_str())); - for (unsigned nsi = 0; nsi < extensions.size(); ++nsi) { - const WFSTNode* next_r = extensions[nsi].first; - const EGrammarNode* next_dot = &git->second; - const bool grammar_continues = next_dot->GrammarContinues(); - const bool rule_completes = next_dot->RuleCompletes(); - if (extensions[nsi].second) - cerr << "!!! " << extensions[nsi].second->AsString() << endl; - // cerr << " rule completes: " << rule_completes << " after consuming " << TD::Convert(git->first) << endl; - assert(grammar_continues || rule_completes); - const SparseVector<double>& input_features = next_dot->GetCFGProductionFeatures(); - if (rule_completes) - IncorporateNewEdge(new Edge(edge->cat, next_dot, edge->q, next_r, edge, extensions[nsi].second, input_features)); - if (grammar_continues) - IncorporateNewEdge(new Edge(edge->cat, next_dot, edge->q, next_r, edge, extensions[nsi].second)); - } - } - } - - void DoPredict(const Edge* edge, const EGrammar& g) { - const EGrammarNode* dot = edge->dot; - const map<WordID, EGrammarNode>& non_terms = dot->GetNonTerminals(); - for (map<WordID, EGrammarNode>::const_iterator git = non_terms.begin(); - git != non_terms.end(); ++git) { - const WordID nt_to_predict = git->first; - //cerr << edge->id << " -- " << TD::Convert(nt_to_predict*-1) << endl; - EGrammar::const_iterator egi = g.find(nt_to_predict); - if (egi == g.end()) { - cerr << "[ERROR] Can't find any grammar rules with a LHS of type " - << TD::Convert(-1*nt_to_predict) << '!' << endl; - continue; - } - assert(edge->IsActive()); - const EGrammarNode* new_dot = &egi->second; - Edge* new_edge = new Edge(nt_to_predict, new_dot, edge->r, edge); - IncorporateNewEdge(new_edge); - } - } - - void DoComplete(const Edge* passive) { -#ifdef DEBUG_CHART_PARSER - cerr << " complete: " << *passive << endl; -#endif - const WordID completed_nt = passive->cat; - const WFSTNode* q = passive->q; - const WFSTNode* next_r = passive->r; - const Edge query(q); - const pair<unordered_multiset<const Edge*, REdgeHash, REdgeEquals>::iterator, - unordered_multiset<const Edge*, REdgeHash, REdgeEquals>::iterator > p = - active_edges.equal_range(&query); - for (unordered_multiset<const Edge*, REdgeHash, REdgeEquals>::iterator it = p.first; - it != p.second; ++it) { - const Edge* active = *it; -#ifdef DEBUG_CHART_PARSER - cerr << " pos: " << *active << endl; -#endif - const EGrammarNode* next_dot = active->dot->Extend(completed_nt); - if (!next_dot) continue; - const SparseVector<double>& input_features = next_dot->GetCFGProductionFeatures(); - // add up to 2 rules - if (next_dot->RuleCompletes()) - IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive, input_features)); - if (next_dot->GrammarContinues()) - IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive)); - } - } - - void DoMergeWithPassives(const Edge* active) { - // edge is active, has non-terminals, we need to find the passives that can extend it - assert(active->IsActive()); - assert(active->dot->HasNonTerminals()); -#ifdef DEBUG_CHART_PARSER - cerr << " merge active with passives: ACT=" << *active << endl; -#endif - const Edge query(active->r, 1); - const pair<unordered_multiset<const Edge*, QEdgeHash, QEdgeEquals>::iterator, - unordered_multiset<const Edge*, QEdgeHash, QEdgeEquals>::iterator > p = - passive_edges.equal_range(&query); - for (unordered_multiset<const Edge*, QEdgeHash, QEdgeEquals>::iterator it = p.first; - it != p.second; ++it) { - const Edge* passive = *it; - const EGrammarNode* next_dot = active->dot->Extend(passive->cat); - if (!next_dot) continue; - const WFSTNode* next_r = passive->r; - const SparseVector<double>& input_features = next_dot->GetCFGProductionFeatures(); - if (next_dot->RuleCompletes()) - IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive, input_features)); - if (next_dot->GrammarContinues()) - IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive)); - } - } - - // take ownership of edge memory, add to various indexes, etc - // returns true if this edge is new - bool IncorporateNewEdge(Edge* edge) { - free_list_.push_back(edge); - if (edge->passive_parent && edge->active_parent) { - Traversal* t = new Traversal(edge, edge->active_parent, edge->passive_parent); - traversal_free_list_.push_back(t); - if (all_traversals.find(t) != all_traversals.end()) { - return false; - } else { - all_traversals.insert(t); - } - } - exp_agenda.AddEdge(edge); - return true; - } - - bool FinishEdge(const Edge* edge, Hypergraph* hg) { - bool is_new = false; - if (all_edges.find(edge) == all_edges.end()) { -#ifdef DEBUG_CHART_PARSER - cerr << *edge << " is NEW\n"; -#endif - all_edges.insert(edge); - is_new = true; - if (edge->IsPassive()) passive_edges.insert(edge); - if (edge->IsActive()) active_edges.insert(edge); - agenda.AddEdge(edge); - } else { -#ifdef DEBUG_CHART_PARSER - cerr << *edge << " is NOT NEW.\n"; -#endif - } - AddEdgeToTranslationForest(edge, hg); - return is_new; - } - - // build the translation forest - void AddEdgeToTranslationForest(const Edge* edge, Hypergraph* hg) { - assert(hg->nodes_.size() < kMAX_NODES); - Hypergraph::Node* tps = NULL; - // first add any target language rules - if (edge->tps) { - Hypergraph::Node*& node = tps2node[(size_t)edge->tps.get()]; - if (!node) { - // cerr << "Creating phrases for " << edge->tps << endl; - const TRulePtr& rule = edge->tps; - node = hg->AddNode(kPHRASE); - Hypergraph::Edge* hg_edge = hg->AddEdge(rule, Hypergraph::TailNodeVector()); - hg_edge->feature_values_ += rule->GetFeatureValues(); - hg->ConnectEdgeToHeadNode(hg_edge, node); - } - tps = node; - } - Hypergraph::Node*& head_node = edge2node[edge]; - if (!head_node) - head_node = hg->AddNode(kPHRASE); - if (edge->cat == start_cat_ && edge->q == q_0_ && edge->r == q_final_ && edge->IsPassive()) { - assert(goal_node == NULL || goal_node == head_node); - goal_node = head_node; - } - Hypergraph::TailNodeVector tail; - SparseVector<double> extra; - if (edge->IsCreatedByPredict()) { - // extra.set_value(FD::Convert("predict"), 1); - } else if (edge->IsCreatedByScan()) { - tail.push_back(edge2node[edge->active_parent]->id_); - if (tps) { - tail.push_back(tps->id_); - } - //extra.set_value(FD::Convert("scan"), 1); - } else if (edge->IsCreatedByComplete()) { - tail.push_back(edge2node[edge->active_parent]->id_); - tail.push_back(edge2node[edge->passive_parent]->id_); - //extra.set_value(FD::Convert("complete"), 1); - } else { - assert(!"unexpected edge type!"); - } - //cerr << head_node->id_ << "<--" << *edge << endl; - -#ifdef DEBUG_CHART_PARSER - for (int i = 0; i < tail.size(); ++i) - if (tail[i] == head_node->id_) { - cerr << "ERROR: " << *edge << "\n i=" << i << endl; - if (i == 1) { cerr << "\tP: " << *edge->passive_parent << endl; } - if (i == 0) { cerr << "\tA: " << *edge->active_parent << endl; } - assert(!"self-loop found!"); - } -#endif - Hypergraph::Edge* hg_edge = NULL; - if (tail.size() == 0) { - hg_edge = hg->AddEdge(kEPSRule, tail); - } else if (tail.size() == 1) { - hg_edge = hg->AddEdge(kX1, tail); - } else if (tail.size() == 2) { - hg_edge = hg->AddEdge(kX1X2, tail); - } - if (edge->features) - hg_edge->feature_values_ += *edge->features; - hg_edge->feature_values_ += extra; - hg->ConnectEdgeToHeadNode(hg_edge, head_node); - } - - Hypergraph::Node* goal_node; - EdgeQueue exp_agenda; - EdgeQueue agenda; - unordered_map<size_t, Hypergraph::Node*> tps2node; - unordered_map<const Edge*, Hypergraph::Node*, UniqueEdgeHash, UniqueEdgeEquals> edge2node; - unordered_set<const Traversal*, UniqueTraversalHash, UniqueTraversalEquals> all_traversals; - unordered_set<const Edge*, UniqueEdgeHash, UniqueEdgeEquals> all_edges; - unordered_multiset<const Edge*, QEdgeHash, QEdgeEquals> passive_edges; - unordered_multiset<const Edge*, REdgeHash, REdgeEquals> active_edges; - vector<Edge*> free_list_; - vector<Traversal*> traversal_free_list_; - const WordID start_cat_; - const WFSTNode* const q_0_; - const WFSTNode* const q_final_; -}; - -#ifdef DEBUG_CHART_PARSER -static string TrimRule(const string& r) { - size_t start = r.find(" |||") + 5; - size_t end = r.rfind(" |||"); - return r.substr(start, end - start); -} -#endif - -void AddGrammarRule(const string& r, EGrammar* g) { - const size_t pos = r.find(" ||| "); - if (pos == string::npos || r[0] != '[') { - cerr << "Bad rule: " << r << endl; - return; - } - const size_t rpos = r.rfind(" ||| "); - string feats; - string rs = r; - if (rpos != pos) { - feats = r.substr(rpos + 5); - rs = r.substr(0, rpos); - } - string rhs = rs.substr(pos + 5); - string trule = rs + " ||| " + rhs + " ||| " + feats; - TRule tr(trule); - cerr << "X: " << tr.e_[0] << endl; -#ifdef DEBUG_CHART_PARSER - string hint_last_rule; -#endif - EGrammarNode* cur = &(*g)[tr.GetLHS()]; - cur->is_root = true; - for (int i = 0; i < tr.FLength(); ++i) { - WordID sym = tr.f()[i]; -#ifdef DEBUG_CHART_PARSER - hint_last_rule = TD::Convert(sym < 0 ? -sym : sym); - cur->hint += " <@@> (*" + hint_last_rule + ") " + TrimRule(tr.AsString()); -#endif - if (sym < 0) - cur = &cur->ntptr[sym]; - else - cur = &cur->tptr[sym]; - } -#ifdef DEBUG_CHART_PARSER - cur->hint += " <@@> (" + hint_last_rule + "*) " + TrimRule(tr.AsString()); -#endif - cur->is_some_rule_complete = true; - cur->input_features = tr.GetFeatureValues(); -} - -CFG_WFSTComposer::~CFG_WFSTComposer() { - delete pimpl_; -} - -CFG_WFSTComposer::CFG_WFSTComposer(const WFST& wfst) { - InitializeConstants(); - pimpl_ = new CFG_WFSTComposerImpl(kUNIQUE_START, wfst.Initial(), wfst.Final()); -} - -bool CFG_WFSTComposer::Compose(const Hypergraph& src_forest, Hypergraph* trg_forest) { - // first, convert the src forest into an EGrammar - EGrammar g; - const int nedges = src_forest.edges_.size(); - const int nnodes = src_forest.nodes_.size(); - vector<int> cats(nnodes); - bool assign_cats = false; - for (int i = 0; i < nnodes; ++i) - if (assign_cats) { - cats[i] = TD::Convert("CAT_" + boost::lexical_cast<string>(i)) * -1; - } else { - cats[i] = src_forest.nodes_[i].cat_; - } - // construct the grammar - for (int i = 0; i < nedges; ++i) { - const Hypergraph::Edge& edge = src_forest.edges_[i]; - const vector<WordID>& src = edge.rule_->f(); - EGrammarNode* cur = &g[cats[edge.head_node_]]; - cur->is_root = true; - int ntc = 0; - for (int j = 0; j < src.size(); ++j) { - WordID sym = src[j]; - if (sym <= 0) { - sym = cats[edge.tail_nodes_[ntc]]; - ++ntc; - cur = &cur->ntptr[sym]; - } else { - cur = &cur->tptr[sym]; - } - } - cur->is_some_rule_complete = true; - cur->input_features = edge.feature_values_; - } - EGrammarNode& goal_rule = g[kUNIQUE_START]; - assert((goal_rule.ntptr.size() == 1 && goal_rule.tptr.size() == 0) || - (goal_rule.ntptr.size() == 0 && goal_rule.tptr.size() == 1)); - - return pimpl_->Compose(g, trg_forest); -} - -bool CFG_WFSTComposer::Compose(istream* in, Hypergraph* trg_forest) { - EGrammar g; - while(*in) { - string line; - getline(*in, line); - if (line.empty()) continue; - AddGrammarRule(line, &g); - } - - return pimpl_->Compose(g, trg_forest); -} diff --git a/gi/pf/cfg_wfst_composer.h b/gi/pf/cfg_wfst_composer.h deleted file mode 100644 index cf47f459..00000000 --- a/gi/pf/cfg_wfst_composer.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef _CFG_WFST_COMPOSER_H_ -#define _CFG_WFST_COMPOSER_H_ - -#include <iostream> -#include <vector> -#include <utility> - -#include "trule.h" -#include "wordid.h" - -class CFG_WFSTComposerImpl; -class Hypergraph; - -struct WFSTNode { - virtual ~WFSTNode(); - // returns the next states reachable by consuming srcindex (which identifies a word) - // paired with the output string generated by taking that transition. - virtual std::vector<std::pair<const WFSTNode*,TRulePtr> > ExtendInput(unsigned srcindex) const = 0; -}; - -struct WFST { - virtual ~WFST(); - virtual const WFSTNode* Final() const = 0; - virtual const WFSTNode* Initial() const = 0; -}; - -class CFG_WFSTComposer { - public: - ~CFG_WFSTComposer(); - explicit CFG_WFSTComposer(const WFST& wfst); - bool Compose(const Hypergraph& in_forest, Hypergraph* trg_forest); - - // reads the grammar from a file. There must be a single top-level - // S -> X rule. Anything else is possible. Format is: - // [S] ||| [SS,1] - // [SS] ||| [NP,1] [VP,2] ||| Feature1=0.2 Feature2=-2.3 - // [SS] ||| [VP,1] [NP,2] ||| Feature1=0.8 - // [NP] ||| [DET,1] [N,2] ||| Feature3=2 - // ... - bool Compose(std::istream* grammar_file, Hypergraph* trg_forest); - - private: - CFG_WFSTComposerImpl* pimpl_; -}; - -#endif diff --git a/gi/pf/conditional_pseg.h b/gi/pf/conditional_pseg.h deleted file mode 100644 index 81ddb206..00000000 --- a/gi/pf/conditional_pseg.h +++ /dev/null @@ -1,275 +0,0 @@ -#ifndef _CONDITIONAL_PSEG_H_ -#define _CONDITIONAL_PSEG_H_ - -#include <vector> -#include <tr1/unordered_map> -#include <boost/functional/hash.hpp> -#include <iostream> - -#include "m.h" -#include "prob.h" -#include "ccrp_nt.h" -#include "mfcr.h" -#include "trule.h" -#include "base_distributions.h" -#include "tdict.h" - -template <typename ConditionalBaseMeasure> -struct MConditionalTranslationModel { - explicit MConditionalTranslationModel(ConditionalBaseMeasure& rcp0) : - rp0(rcp0), d(0.5), strength(1.0), lambdas(1, prob_t::One()), p0s(1) {} - - void Summary() const { - std::cerr << "Number of conditioning contexts: " << r.size() << std::endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - std::cerr << TD::GetString(it->first) << " \t(d=" << it->second.discount() << ",s=" << it->second.strength() << ") --------------------------" << std::endl; - for (MFCR<1,TRule>::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - std::cerr << " " << i2->second.total_dish_count_ << '\t' << i2->first << std::endl; - } - } - - double log_likelihood(const double& dd, const double& aa) const { - if (aa <= -dd) return -std::numeric_limits<double>::infinity(); - //double llh = Md::log_beta_density(dd, 10, 3) + Md::log_gamma_density(aa, 1, 1); - double llh = Md::log_beta_density(dd, 1, 1) + - Md::log_gamma_density(dd + aa, 1, 1); - typename std::tr1::unordered_map<std::vector<WordID>, MFCR<1,TRule>, boost::hash<std::vector<WordID> > >::const_iterator it; - for (it = r.begin(); it != r.end(); ++it) - llh += it->second.log_crp_prob(dd, aa); - return llh; - } - - struct DiscountResampler { - DiscountResampler(const MConditionalTranslationModel& m) : m_(m) {} - const MConditionalTranslationModel& m_; - double operator()(const double& proposed_discount) const { - return m_.log_likelihood(proposed_discount, m_.strength); - } - }; - - struct AlphaResampler { - AlphaResampler(const MConditionalTranslationModel& m) : m_(m) {} - const MConditionalTranslationModel& m_; - double operator()(const double& proposed_strength) const { - return m_.log_likelihood(m_.d, proposed_strength); - } - }; - - void ResampleHyperparameters(MT19937* rng) { - typename std::tr1::unordered_map<std::vector<WordID>, MFCR<1,TRule>, boost::hash<std::vector<WordID> > >::iterator it; -#if 1 - for (it = r.begin(); it != r.end(); ++it) { - it->second.resample_hyperparameters(rng); - } -#else - const unsigned nloop = 5; - const unsigned niterations = 10; - DiscountResampler dr(*this); - AlphaResampler ar(*this); - for (int iter = 0; iter < nloop; ++iter) { - strength = slice_sampler1d(ar, strength, *rng, -d + std::numeric_limits<double>::min(), - std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); - double min_discount = std::numeric_limits<double>::min(); - if (strength < 0.0) min_discount -= strength; - d = slice_sampler1d(dr, d, *rng, min_discount, - 1.0, 0.0, niterations, 100*niterations); - } - strength = slice_sampler1d(ar, strength, *rng, -d, - std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); - std::cerr << "MConditionalTranslationModel(d=" << d << ",s=" << strength << ") = " << log_likelihood(d, strength) << std::endl; - for (it = r.begin(); it != r.end(); ++it) { - it->second.set_discount(d); - it->second.set_strength(strength); - } -#endif - } - - int DecrementRule(const TRule& rule, MT19937* rng) { - RuleModelHash::iterator it = r.find(rule.f_); - assert(it != r.end()); - const TableCount delta = it->second.decrement(rule, rng); - if (delta.count) { - if (it->second.num_customers() == 0) r.erase(it); - } - return delta.count; - } - - int IncrementRule(const TRule& rule, MT19937* rng) { - RuleModelHash::iterator it = r.find(rule.f_); - if (it == r.end()) { - //it = r.insert(make_pair(rule.f_, MFCR<1,TRule>(d, strength))).first; - it = r.insert(make_pair(rule.f_, MFCR<1,TRule>(1,1,1,1,0.6, -0.12))).first; - } - p0s[0] = rp0(rule); - TableCount delta = it->second.increment(rule, p0s.begin(), lambdas.begin(), rng); - return delta.count; - } - - prob_t RuleProbability(const TRule& rule) const { - prob_t p; - RuleModelHash::const_iterator it = r.find(rule.f_); - if (it == r.end()) { - p = rp0(rule); - } else { - p0s[0] = rp0(rule); - p = it->second.prob(rule, p0s.begin(), lambdas.begin()); - } - return p; - } - - prob_t Likelihood() const { - prob_t p; p.logeq(log_likelihood(d, strength)); - return p; - } - - const ConditionalBaseMeasure& rp0; - typedef std::tr1::unordered_map<std::vector<WordID>, - MFCR<1, TRule>, - boost::hash<std::vector<WordID> > > RuleModelHash; - RuleModelHash r; - double d, strength; - std::vector<prob_t> lambdas; - mutable std::vector<prob_t> p0s; -}; - -template <typename ConditionalBaseMeasure> -struct ConditionalTranslationModel { - explicit ConditionalTranslationModel(ConditionalBaseMeasure& rcp0) : - rp0(rcp0) {} - - void Summary() const { - std::cerr << "Number of conditioning contexts: " << r.size() << std::endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - std::cerr << TD::GetString(it->first) << " \t(\\alpha = " << it->second.alpha() << ") --------------------------" << std::endl; - for (CCRP_NoTable<TRule>::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - std::cerr << " " << i2->second << '\t' << i2->first << std::endl; - } - } - - void ResampleHyperparameters(MT19937* rng) { - for (RuleModelHash::iterator it = r.begin(); it != r.end(); ++it) - it->second.resample_hyperparameters(rng); - } - - int DecrementRule(const TRule& rule) { - RuleModelHash::iterator it = r.find(rule.f_); - assert(it != r.end()); - int count = it->second.decrement(rule); - if (count) { - if (it->second.num_customers() == 0) r.erase(it); - } - return count; - } - - int IncrementRule(const TRule& rule) { - RuleModelHash::iterator it = r.find(rule.f_); - if (it == r.end()) { - it = r.insert(make_pair(rule.f_, CCRP_NoTable<TRule>(1.0, 1.0, 8.0))).first; - } - int count = it->second.increment(rule); - return count; - } - - void IncrementRules(const std::vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const std::vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - prob_t RuleProbability(const TRule& rule) const { - prob_t p; - RuleModelHash::const_iterator it = r.find(rule.f_); - if (it == r.end()) { - p.logeq(log(rp0(rule))); - } else { - p.logeq(it->second.logprob(rule, log(rp0(rule)))); - } - return p; - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - prob_t q; q.logeq(it->second.log_crp_prob()); - p *= q; - for (CCRP_NoTable<TRule>::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - p *= rp0(i2->first); - } - return p; - } - - const ConditionalBaseMeasure& rp0; - typedef std::tr1::unordered_map<std::vector<WordID>, - CCRP_NoTable<TRule>, - boost::hash<std::vector<WordID> > > RuleModelHash; - RuleModelHash r; -}; - -template <typename ConditionalBaseMeasure> -struct ConditionalParallelSegementationModel { - explicit ConditionalParallelSegementationModel(ConditionalBaseMeasure& rcp0) : - tmodel(rcp0), base(prob_t::One()), aligns(1,1) {} - - ConditionalTranslationModel<ConditionalBaseMeasure> tmodel; - - void DecrementRule(const TRule& rule) { - tmodel.DecrementRule(rule); - } - - void IncrementRule(const TRule& rule) { - tmodel.IncrementRule(rule); - } - - void IncrementRulesAndAlignments(const std::vector<TRulePtr>& rules) { - tmodel.IncrementRules(rules); - for (int i = 0; i < rules.size(); ++i) { - IncrementAlign(rules[i]->f_.size()); - } - } - - void DecrementRulesAndAlignments(const std::vector<TRulePtr>& rules) { - tmodel.DecrementRules(rules); - for (int i = 0; i < rules.size(); ++i) { - DecrementAlign(rules[i]->f_.size()); - } - } - - prob_t RuleProbability(const TRule& rule) const { - return tmodel.RuleProbability(rule); - } - - void IncrementAlign(unsigned span) { - if (aligns.increment(span)) { - // TODO - } - } - - void DecrementAlign(unsigned span) { - if (aligns.decrement(span)) { - // TODO - } - } - - prob_t AlignProbability(unsigned span) const { - prob_t p; - p.logeq(aligns.logprob(span, Md::log_poisson(span, 1.0))); - return p; - } - - prob_t Likelihood() const { - prob_t p; p.logeq(aligns.log_crp_prob()); - p *= base; - p *= tmodel.Likelihood(); - return p; - } - - prob_t base; - CCRP_NoTable<unsigned> aligns; -}; - -#endif - diff --git a/gi/pf/condnaive.cc b/gi/pf/condnaive.cc deleted file mode 100644 index 419731ac..00000000 --- a/gi/pf/condnaive.cc +++ /dev/null @@ -1,298 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/multi_array.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "base_distributions.h" -#include "monotonic_pseg.h" -#include "conditional_pseg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "corpus.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -static unsigned kMAX_SRC_PHRASE; -static unsigned kMAX_TRG_PHRASE; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("input,i",po::value<string>(),"Read parallel data from") - ("max_src_phrase",po::value<unsigned>()->default_value(4),"Maximum length of source language phrases") - ("max_trg_phrase",po::value<unsigned>()->default_value(4),"Maximum length of target language phrases") - ("model1,m",po::value<string>(),"Model 1 parameters (used in base distribution)") - ("model1_interpolation_weight",po::value<double>()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -boost::shared_ptr<MT19937> prng; - -struct ModelAndData { - explicit ModelAndData(ConditionalParallelSegementationModel<PhraseConditionalBase>& m, const vector<vector<int> >& ce, const vector<vector<int> >& cf, const set<int>& ve, const set<int>& vf) : - model(m), - rng(&*prng), - corpuse(ce), - corpusf(cf), - vocabe(ve), - vocabf(vf), - mh_samples(), - mh_rejects(), - kX(-TD::Convert("X")), - derivations(corpuse.size()) {} - - void ResampleHyperparameters() { - } - - void InstantiateRule(const pair<short,short>& from, - const pair<short,short>& to, - const vector<int>& sentf, - const vector<int>& sente, - TRule* rule) const { - rule->f_.clear(); - rule->e_.clear(); - rule->lhs_ = kX; - for (short i = from.first; i < to.first; ++i) - rule->f_.push_back(sentf[i]); - for (short i = from.second; i < to.second; ++i) - rule->e_.push_back(sente[i]); - } - - void DecrementDerivation(const vector<pair<short,short> >& d, const vector<int>& sentf, const vector<int>& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - model.DecrementRule(x); - model.DecrementAlign(x.f_.size()); - } - } - - void PrintDerivation(const vector<pair<short,short> >& d, const vector<int>& sentf, const vector<int>& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - cerr << i << '/' << (d.size() - 1) << ": " << x << endl; - } - } - - void IncrementDerivation(const vector<pair<short,short> >& d, const vector<int>& sentf, const vector<int>& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - model.IncrementRule(x); - model.IncrementAlign(x.f_.size()); - } - } - - prob_t Likelihood() const { - return model.Likelihood(); - } - - prob_t DerivationProposalProbability(const vector<pair<short,short> >& d, const vector<int>& sentf, const vector<int>& sente) const { - prob_t p = prob_t::One(); - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - p *= model.RuleProbability(x); - p *= model.AlignProbability(x.f_.size()); - } - return p; - } - - void Sample(); - - ConditionalParallelSegementationModel<PhraseConditionalBase>& model; - MT19937* rng; - const vector<vector<int> >& corpuse, corpusf; - const set<int>& vocabe, vocabf; - unsigned mh_samples, mh_rejects; - const int kX; - vector<vector<pair<short, short> > > derivations; -}; - -void ModelAndData::Sample() { - unsigned MAXK = kMAX_SRC_PHRASE; - unsigned MAXL = kMAX_TRG_PHRASE; - TRule x; - x.lhs_ = -TD::Convert("X"); - - for (int samples = 0; samples < 1000; ++samples) { - if (samples % 1 == 0 && samples > 0) { - //ResampleHyperparameters(); - cerr << " [" << samples << " LLH=" << log(Likelihood()) << " MH=" << ((double)mh_rejects / mh_samples) << "]\n"; - for (int i = 0; i < 10; ++i) { - cerr << "SENTENCE: " << TD::GetString(corpusf[i]) << " ||| " << TD::GetString(corpuse[i]) << endl; - PrintDerivation(derivations[i], corpusf[i], corpuse[i]); - } - static TRule xx("[X] ||| w n ||| s h ||| X=0"); - const CCRP_NoTable<TRule>& dcrp = model.tmodel.r.find(xx.f_)->second; - for (CCRP_NoTable<TRule>::const_iterator it = dcrp.begin(); it != dcrp.end(); ++it) { - cerr << "\t" << it->second << "\t" << it->first << endl; - } - } - cerr << '.' << flush; - for (int s = 0; s < corpuse.size(); ++s) { - const vector<int>& sentf = corpusf[s]; - const vector<int>& sente = corpuse[s]; -// cerr << " CUSTOMERS: " << rules.num_customers() << endl; -// cerr << "SENTENCE: " << TD::GetString(sentf) << " ||| " << TD::GetString(sente) << endl; - - vector<pair<short, short> >& deriv = derivations[s]; - const prob_t p_cur = Likelihood(); - DecrementDerivation(deriv, sentf, sente); - - boost::multi_array<prob_t, 2> a(boost::extents[sentf.size() + 1][sente.size() + 1]); - boost::multi_array<prob_t, 4> trans(boost::extents[sentf.size() + 1][sente.size() + 1][MAXK][MAXL]); - a[0][0] = prob_t::One(); - for (int i = 0; i < sentf.size(); ++i) { - for (int j = 0; j < sente.size(); ++j) { - const prob_t src_a = a[i][j]; - x.f_.clear(); - for (int k = 1; k <= MAXK; ++k) { - if (i + k > sentf.size()) break; - x.f_.push_back(sentf[i + k - 1]); - x.e_.clear(); - const prob_t p_span = model.AlignProbability(k); // prob of consuming this much source - for (int l = 1; l <= MAXL; ++l) { - if (j + l > sente.size()) break; - x.e_.push_back(sente[j + l - 1]); - trans[i][j][k - 1][l - 1] = model.RuleProbability(x) * p_span; - a[i + k][j + l] += src_a * trans[i][j][k - 1][l - 1]; - } - } - } - } -// cerr << "Inside: " << log(a[sentf.size()][sente.size()]) << endl; - const prob_t q_cur = DerivationProposalProbability(deriv, sentf, sente); - - vector<pair<short,short> > newderiv; - int cur_i = sentf.size(); - int cur_j = sente.size(); - while(cur_i > 0 && cur_j > 0) { - newderiv.push_back(pair<short,short>(cur_i, cur_j)); -// cerr << "NODE: (" << cur_i << "," << cur_j << ")\n"; - SampleSet<prob_t> ss; - vector<pair<short,short> > nexts; - for (int k = 1; k <= MAXK; ++k) { - const int hyp_i = cur_i - k; - if (hyp_i < 0) break; - for (int l = 1; l <= MAXL; ++l) { - const int hyp_j = cur_j - l; - if (hyp_j < 0) break; - const prob_t& inside = a[hyp_i][hyp_j]; - if (inside == prob_t::Zero()) continue; - const prob_t& transp = trans[hyp_i][hyp_j][k - 1][l - 1]; - if (transp == prob_t::Zero()) continue; - const prob_t p = inside * transp; - ss.add(p); - nexts.push_back(pair<short,short>(hyp_i, hyp_j)); -// cerr << " (" << hyp_i << "," << hyp_j << ") <--- " << log(p) << endl; - } - } -// cerr << " sample set has " << nexts.size() << " elements.\n"; - const int selected = rng->SelectSample(ss); - cur_i = nexts[selected].first; - cur_j = nexts[selected].second; - } - newderiv.push_back(pair<short,short>(0,0)); - const prob_t q_new = DerivationProposalProbability(newderiv, sentf, sente); - IncrementDerivation(newderiv, sentf, sente); -// cerr << "SANITY: " << q_new << " " <<log(DerivationProposalProbability(newderiv, sentf, sente)) << endl; - if (deriv.empty()) { deriv = newderiv; continue; } - ++mh_samples; - - if (deriv != newderiv) { - const prob_t p_new = Likelihood(); -// cerr << "p_cur=" << log(p_cur) << "\t p_new=" << log(p_new) << endl; -// cerr << "q_cur=" << log(q_cur) << "\t q_new=" << log(q_new) << endl; - if (!rng->AcceptMetropolisHastings(p_new, p_cur, q_new, q_cur)) { - ++mh_rejects; - DecrementDerivation(newderiv, sentf, sente); - IncrementDerivation(deriv, sentf, sente); - } else { -// cerr << " ACCEPT\n"; - deriv = newderiv; - } - } - } - } -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - kMAX_TRG_PHRASE = conf["max_trg_phrase"].as<unsigned>(); - kMAX_SRC_PHRASE = conf["max_src_phrase"].as<unsigned>(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); -// MT19937& rng = *prng; - - vector<vector<int> > corpuse, corpusf; - set<int> vocabe, vocabf; - corpus::ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - Model1 m1(conf["model1"].as<string>()); - - PhraseConditionalBase pcb0(m1, conf["model1_interpolation_weight"].as<double>(), vocabe.size()); - ConditionalParallelSegementationModel<PhraseConditionalBase> x(pcb0); - - ModelAndData posterior(x, corpuse, corpusf, vocabe, vocabf); - posterior.Sample(); - - TRule r1("[X] ||| x ||| l e ||| X=0"); - TRule r2("[X] ||| A ||| a d ||| X=0"); - TRule r3("[X] ||| n ||| e r ||| X=0"); - TRule r4("[X] ||| x A n ||| b l a g ||| X=0"); - - PhraseConditionalUninformativeBase u0(vocabe.size()); - - cerr << (pcb0(r1)*pcb0(r2)*pcb0(r3)) << endl; - cerr << (u0(r4)) << endl; - - return 0; -} - diff --git a/gi/pf/corpus.cc b/gi/pf/corpus.cc deleted file mode 100644 index cb6e4ed7..00000000 --- a/gi/pf/corpus.cc +++ /dev/null @@ -1,62 +0,0 @@ -#include "corpus.h" - -#include <set> -#include <vector> -#include <string> - -#include "tdict.h" -#include "filelib.h" - -using namespace std; - -namespace corpus { - -void ReadParallelCorpus(const string& filename, - vector<vector<WordID> >* f, - vector<vector<WordID> >* e, - set<WordID>* vocab_f, - set<WordID>* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - ReadFile rf(filename); - istream* in = rf.stream(); - assert(*in); - string line; - unsigned lc = 0; - const WordID kDIV = TD::Convert("|||"); - vector<WordID> tmp; - while(getline(*in, line)) { - ++lc; - e->push_back(vector<int>()); - f->push_back(vector<int>()); - vector<int>& le = e->back(); - vector<int>& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { - isf = false; - } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - if (cur == kDIV) { - cerr << "ERROR in " << lc << ": " << line << endl << endl; - abort(); - } - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } -} - -} - diff --git a/gi/pf/corpus.h b/gi/pf/corpus.h deleted file mode 100644 index e7febdb7..00000000 --- a/gi/pf/corpus.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef _CORPUS_H_ -#define _CORPUS_H_ - -#include <string> -#include <vector> -#include <set> -#include "wordid.h" - -namespace corpus { - -void ReadParallelCorpus(const std::string& filename, - std::vector<std::vector<WordID> >* f, - std::vector<std::vector<WordID> >* e, - std::set<WordID>* vocab_f, - std::set<WordID>* vocab_e); - -} - -#endif diff --git a/gi/pf/dpnaive.cc b/gi/pf/dpnaive.cc deleted file mode 100644 index 75ccad72..00000000 --- a/gi/pf/dpnaive.cc +++ /dev/null @@ -1,301 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/multi_array.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "base_distributions.h" -#include "monotonic_pseg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "corpus.h" - -using namespace std; -using namespace std::tr1; -namespace po = boost::program_options; - -static unsigned kMAX_SRC_PHRASE; -static unsigned kMAX_TRG_PHRASE; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("input,i",po::value<string>(),"Read parallel data from") - ("max_src_phrase",po::value<unsigned>()->default_value(4),"Maximum length of source language phrases") - ("max_trg_phrase",po::value<unsigned>()->default_value(4),"Maximum length of target language phrases") - ("model1,m",po::value<string>(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value<string>(),"Inverse Model 1 parameters (used in base distribution)") - ("model1_interpolation_weight",po::value<double>()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -boost::shared_ptr<MT19937> prng; - -template <typename Base> -struct ModelAndData { - explicit ModelAndData(MonotonicParallelSegementationModel<PhraseJointBase_BiDir>& m, const Base& b, const vector<vector<int> >& ce, const vector<vector<int> >& cf, const set<int>& ve, const set<int>& vf) : - model(m), - rng(&*prng), - p0(b), - baseprob(prob_t::One()), - corpuse(ce), - corpusf(cf), - vocabe(ve), - vocabf(vf), - mh_samples(), - mh_rejects(), - kX(-TD::Convert("X")), - derivations(corpuse.size()) {} - - void ResampleHyperparameters() { - } - - void InstantiateRule(const pair<short,short>& from, - const pair<short,short>& to, - const vector<int>& sentf, - const vector<int>& sente, - TRule* rule) const { - rule->f_.clear(); - rule->e_.clear(); - rule->lhs_ = kX; - for (short i = from.first; i < to.first; ++i) - rule->f_.push_back(sentf[i]); - for (short i = from.second; i < to.second; ++i) - rule->e_.push_back(sente[i]); - } - - void DecrementDerivation(const vector<pair<short,short> >& d, const vector<int>& sentf, const vector<int>& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - model.DecrementRule(x); - model.DecrementContinue(); - } - model.DecrementStop(); - } - - void PrintDerivation(const vector<pair<short,short> >& d, const vector<int>& sentf, const vector<int>& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - cerr << i << '/' << (d.size() - 1) << ": " << x << endl; - } - } - - void IncrementDerivation(const vector<pair<short,short> >& d, const vector<int>& sentf, const vector<int>& sente) { - if (d.size() < 2) return; - TRule x; - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - model.IncrementRule(x); - model.IncrementContinue(); - } - model.IncrementStop(); - } - - prob_t Likelihood() const { - return model.Likelihood(); - } - - prob_t DerivationProposalProbability(const vector<pair<short,short> >& d, const vector<int>& sentf, const vector<int>& sente) const { - prob_t p = model.StopProbability(); - if (d.size() < 2) return p; - TRule x; - const prob_t p_cont = model.ContinueProbability(); - for (int i = 1; i < d.size(); ++i) { - InstantiateRule(d[i], d[i-1], sentf, sente, &x); - p *= p_cont; - p *= model.RuleProbability(x); - } - return p; - } - - void Sample(); - - MonotonicParallelSegementationModel<PhraseJointBase_BiDir>& model; - MT19937* rng; - const Base& p0; - prob_t baseprob; // cached value of generating the table table labels from p0 - // this can't be used if we go to a hierarchical prior! - const vector<vector<int> >& corpuse, corpusf; - const set<int>& vocabe, vocabf; - unsigned mh_samples, mh_rejects; - const int kX; - vector<vector<pair<short, short> > > derivations; -}; - -template <typename Base> -void ModelAndData<Base>::Sample() { - unsigned MAXK = kMAX_SRC_PHRASE; - unsigned MAXL = kMAX_TRG_PHRASE; - TRule x; - x.lhs_ = -TD::Convert("X"); - for (int samples = 0; samples < 1000; ++samples) { - if (samples % 1 == 0 && samples > 0) { - //ResampleHyperparameters(); - cerr << " [" << samples << " LLH=" << log(Likelihood()) << " MH=" << ((double)mh_rejects / mh_samples) << "]\n"; - for (int i = 0; i < 10; ++i) { - cerr << "SENTENCE: " << TD::GetString(corpusf[i]) << " ||| " << TD::GetString(corpuse[i]) << endl; - PrintDerivation(derivations[i], corpusf[i], corpuse[i]); - } - } - cerr << '.' << flush; - for (int s = 0; s < corpuse.size(); ++s) { - const vector<int>& sentf = corpusf[s]; - const vector<int>& sente = corpuse[s]; -// cerr << " CUSTOMERS: " << rules.num_customers() << endl; -// cerr << "SENTENCE: " << TD::GetString(sentf) << " ||| " << TD::GetString(sente) << endl; - - vector<pair<short, short> >& deriv = derivations[s]; - const prob_t p_cur = Likelihood(); - DecrementDerivation(deriv, sentf, sente); - - boost::multi_array<prob_t, 2> a(boost::extents[sentf.size() + 1][sente.size() + 1]); - boost::multi_array<prob_t, 4> trans(boost::extents[sentf.size() + 1][sente.size() + 1][MAXK][MAXL]); - a[0][0] = prob_t::One(); - const prob_t q_stop = model.StopProbability(); - const prob_t q_cont = model.ContinueProbability(); - for (int i = 0; i < sentf.size(); ++i) { - for (int j = 0; j < sente.size(); ++j) { - const prob_t src_a = a[i][j]; - x.f_.clear(); - for (int k = 1; k <= MAXK; ++k) { - if (i + k > sentf.size()) break; - x.f_.push_back(sentf[i + k - 1]); - x.e_.clear(); - for (int l = 1; l <= MAXL; ++l) { - if (j + l > sente.size()) break; - x.e_.push_back(sente[j + l - 1]); - const bool stop_now = ((j + l) == sente.size()) && ((i + k) == sentf.size()); - const prob_t& cp = stop_now ? q_stop : q_cont; - trans[i][j][k - 1][l - 1] = model.RuleProbability(x) * cp; - a[i + k][j + l] += src_a * trans[i][j][k - 1][l - 1]; - } - } - } - } -// cerr << "Inside: " << log(a[sentf.size()][sente.size()]) << endl; - const prob_t q_cur = DerivationProposalProbability(deriv, sentf, sente); - - vector<pair<short,short> > newderiv; - int cur_i = sentf.size(); - int cur_j = sente.size(); - while(cur_i > 0 && cur_j > 0) { - newderiv.push_back(pair<short,short>(cur_i, cur_j)); -// cerr << "NODE: (" << cur_i << "," << cur_j << ")\n"; - SampleSet<prob_t> ss; - vector<pair<short,short> > nexts; - for (int k = 1; k <= MAXK; ++k) { - const int hyp_i = cur_i - k; - if (hyp_i < 0) break; - for (int l = 1; l <= MAXL; ++l) { - const int hyp_j = cur_j - l; - if (hyp_j < 0) break; - const prob_t& inside = a[hyp_i][hyp_j]; - if (inside == prob_t::Zero()) continue; - const prob_t& transp = trans[hyp_i][hyp_j][k - 1][l - 1]; - if (transp == prob_t::Zero()) continue; - const prob_t p = inside * transp; - ss.add(p); - nexts.push_back(pair<short,short>(hyp_i, hyp_j)); -// cerr << " (" << hyp_i << "," << hyp_j << ") <--- " << log(p) << endl; - } - } -// cerr << " sample set has " << nexts.size() << " elements.\n"; - const int selected = rng->SelectSample(ss); - cur_i = nexts[selected].first; - cur_j = nexts[selected].second; - } - newderiv.push_back(pair<short,short>(0,0)); - const prob_t q_new = DerivationProposalProbability(newderiv, sentf, sente); - IncrementDerivation(newderiv, sentf, sente); -// cerr << "SANITY: " << q_new << " " <<log(DerivationProposalProbability(newderiv, sentf, sente)) << endl; - if (deriv.empty()) { deriv = newderiv; continue; } - ++mh_samples; - - if (deriv != newderiv) { - const prob_t p_new = Likelihood(); -// cerr << "p_cur=" << log(p_cur) << "\t p_new=" << log(p_new) << endl; -// cerr << "q_cur=" << log(q_cur) << "\t q_new=" << log(q_new) << endl; - if (!rng->AcceptMetropolisHastings(p_new, p_cur, q_new, q_cur)) { - ++mh_rejects; - DecrementDerivation(newderiv, sentf, sente); - IncrementDerivation(deriv, sentf, sente); - } else { -// cerr << " ACCEPT\n"; - deriv = newderiv; - } - } - } - } -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - kMAX_TRG_PHRASE = conf["max_trg_phrase"].as<unsigned>(); - kMAX_SRC_PHRASE = conf["max_src_phrase"].as<unsigned>(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (!conf.count("inverse_model1")) { - cerr << argv[0] << "Please use --inverse_model1 to specify inverse model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); -// MT19937& rng = *prng; - - vector<vector<int> > corpuse, corpusf; - set<int> vocabe, vocabf; - corpus::ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - Model1 m1(conf["model1"].as<string>()); - Model1 invm1(conf["inverse_model1"].as<string>()); -// PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as<double>(), vocabe.size(), vocabf.size()); - PhraseJointBase_BiDir alp0(m1, invm1, conf["model1_interpolation_weight"].as<double>(), vocabe.size(), vocabf.size()); - MonotonicParallelSegementationModel<PhraseJointBase_BiDir> m(alp0); - - ModelAndData<PhraseJointBase_BiDir> posterior(m, alp0, corpuse, corpusf, vocabe, vocabf); - posterior.Sample(); - - return 0; -} - diff --git a/gi/pf/guess-translits.pl b/gi/pf/guess-translits.pl deleted file mode 100755 index d00c2168..00000000 --- a/gi/pf/guess-translits.pl +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use utf8; - -my $MIN_PMI = -3; - -my %fs; -my %es; -my %ef; - -die "Usage: $0 < input.utf8.txt\n" if scalar @ARGV > 0; - -binmode(STDIN,":utf8"); -binmode(STDOUT,":utf8"); -binmode(STDERR,":utf8"); - -my $tot = 0; -print STDERR "Reading alignments from STDIN ...\n"; -while(<STDIN>) { - chomp; - my ($fsent, $esent, $alsent) = split / \|\|\| /; - die "Format should be 'foreign sentence ||| english sentence ||| 0-0 1-1 ...'\n" unless defined $fsent && defined $esent && defined $alsent; - - my @fws = split /\s+/, $fsent; - my @ews = split /\s+/, $esent; - my @as = split /\s+/, $alsent; - my %a2b; - my %b2a; - for my $ap (@as) { - my ($a,$b) = split /-/, $ap; - die "BAD INPUT: $_\n" unless defined $a && defined $b; - $a2b{$a}->{$b} = 1; - $b2a{$b}->{$a} = 1; - } - for my $a (keys %a2b) { - my $bref = $a2b{$a}; - next unless scalar keys %$bref < 2; - my $b = (keys %$bref)[0]; - next unless scalar keys %{$b2a{$b}} < 2; - my $f = $fws[$a]; - next unless defined $f; - next unless length($f) > 3; - my $e = $ews[$b]; - next unless defined $e; - next unless length($e) > 3; - - $ef{$f}->{$e}++; - $es{$e}++; - $fs{$f}++; - $tot++; - } -} -my $ltot = log($tot); -my $num = 0; -print STDERR "Extracting pairs for PMI > $MIN_PMI ...\n"; -for my $f (keys %fs) { - my $logf = log($fs{$f}); - my $esref = $ef{$f}; - for my $e (keys %$esref) { - my $loge = log($es{$e}); - my $ef = $esref->{$e}; - my $logef = log($ef); - my $pmi = $logef - ($loge + $logf); - next if $pmi < $MIN_PMI; - my @flets = split //, $f; - my @elets = split //, $e; - print "@flets ||| @elets\n"; - $num++; - } -} -print STDERR "Extracted $num pairs.\n"; -print STDERR "Recommend running:\n ../../training/model1 -v -d -t -99999 output.txt\n"; diff --git a/gi/pf/hpyp_tm.cc b/gi/pf/hpyp_tm.cc deleted file mode 100644 index f362d3f8..00000000 --- a/gi/pf/hpyp_tm.cc +++ /dev/null @@ -1,133 +0,0 @@ -#include "hpyp_tm.h" - -#include <tr1/unordered_map> -#include <iostream> -#include <queue> - -#include "tdict.h" -#include "ccrp.h" -#include "pyp_word_model.h" -#include "tied_resampler.h" - -using namespace std; -using namespace std::tr1; - -struct FreqBinner { - FreqBinner(const std::string& fname) { fd_.Load(fname); } - unsigned NumberOfBins() const { return fd_.Max() + 1; } - unsigned Bin(const WordID& w) const { return fd_.LookUp(w); } - FreqDict<unsigned> fd_; -}; - -template <typename Base, class Binner = FreqBinner> -struct ConditionalPYPWordModel { - ConditionalPYPWordModel(Base* b, const Binner* bnr = NULL) : - base(*b), - binner(bnr), - btr(binner ? binner->NumberOfBins() + 1u : 2u) {} - - void Summary() const { - cerr << "Number of conditioning contexts: " << r.size() << endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - cerr << TD::Convert(it->first) << " \tPYP(d=" << it->second.discount() << ",s=" << it->second.strength() << ") --------------------------" << endl; - for (CCRP<vector<WordID> >::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - cerr << " " << i2->second << endl; - } - } - - void ResampleHyperparameters(MT19937* rng) { - btr.ResampleHyperparameters(rng); - } - - prob_t Prob(const WordID src, const vector<WordID>& trglets) const { - RuleModelHash::const_iterator it = r.find(src); - if (it == r.end()) { - return base(trglets); - } else { - return it->second.prob(trglets, base(trglets)); - } - } - - void Increment(const WordID src, const vector<WordID>& trglets, MT19937* rng) { - RuleModelHash::iterator it = r.find(src); - if (it == r.end()) { - it = r.insert(make_pair(src, CCRP<vector<WordID> >(0.5,1.0))).first; - static const WordID kNULL = TD::Convert("NULL"); - unsigned bin = (src == kNULL ? 0 : 1); - if (binner && bin) { bin = binner->Bin(src) + 1; } - btr.Add(bin, &it->second); - } - if (it->second.increment(trglets, base(trglets), rng)) - base.Increment(trglets, rng); - } - - void Decrement(const WordID src, const vector<WordID>& trglets, MT19937* rng) { - RuleModelHash::iterator it = r.find(src); - assert(it != r.end()); - if (it->second.decrement(trglets, rng)) { - base.Decrement(trglets, rng); - } - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - prob_t q; q.logeq(it->second.log_crp_prob()); - p *= q; - } - return p; - } - - unsigned UniqueConditioningContexts() const { - return r.size(); - } - - // TODO tie PYP hyperparameters based on source word frequency bins - Base& base; - const Binner* binner; - BinTiedResampler<CCRP<vector<WordID> > > btr; - typedef unordered_map<WordID, CCRP<vector<WordID> > > RuleModelHash; - RuleModelHash r; -}; - -HPYPLexicalTranslation::HPYPLexicalTranslation(const vector<vector<WordID> >& lets, - const unsigned vocab_size, - const unsigned num_letters) : - letters(lets), - base(vocab_size, num_letters, 5), - up0(new PYPWordModel<PoissonUniformWordModel>(&base)), - tmodel(new ConditionalPYPWordModel<PYPWordModel<PoissonUniformWordModel> >(up0, new FreqBinner("10k.freq"))), - kX(-TD::Convert("X")) {} - -void HPYPLexicalTranslation::Summary() const { - tmodel->Summary(); - up0->Summary(); -} - -prob_t HPYPLexicalTranslation::Likelihood() const { - prob_t p = up0->Likelihood(); - p *= tmodel->Likelihood(); - return p; -} - -void HPYPLexicalTranslation::ResampleHyperparameters(MT19937* rng) { - tmodel->ResampleHyperparameters(rng); - up0->ResampleHyperparameters(rng); -} - -unsigned HPYPLexicalTranslation::UniqueConditioningContexts() const { - return tmodel->UniqueConditioningContexts(); -} - -prob_t HPYPLexicalTranslation::Prob(WordID src, WordID trg) const { - return tmodel->Prob(src, letters[trg]); -} - -void HPYPLexicalTranslation::Increment(WordID src, WordID trg, MT19937* rng) { - tmodel->Increment(src, letters[trg], rng); -} - -void HPYPLexicalTranslation::Decrement(WordID src, WordID trg, MT19937* rng) { - tmodel->Decrement(src, letters[trg], rng); -} - diff --git a/gi/pf/hpyp_tm.h b/gi/pf/hpyp_tm.h deleted file mode 100644 index af3215ba..00000000 --- a/gi/pf/hpyp_tm.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef HPYP_LEX_TRANS -#define HPYP_LEX_TRANS - -#include <vector> -#include "wordid.h" -#include "prob.h" -#include "sampler.h" -#include "freqdict.h" -#include "poisson_uniform_word_model.h" - -struct FreqBinner; -template <class B> struct PYPWordModel; -template <typename T, class B> struct ConditionalPYPWordModel; - -struct HPYPLexicalTranslation { - explicit HPYPLexicalTranslation(const std::vector<std::vector<WordID> >& lets, - const unsigned vocab_size, - const unsigned num_letters); - - prob_t Likelihood() const; - - void ResampleHyperparameters(MT19937* rng); - prob_t Prob(WordID src, WordID trg) const; // return p(trg | src) - void Summary() const; - void Increment(WordID src, WordID trg, MT19937* rng); - void Decrement(WordID src, WordID trg, MT19937* rng); - unsigned UniqueConditioningContexts() const; - - private: - const std::vector<std::vector<WordID> >& letters; // spelling dictionary - PoissonUniformWordModel base; // "generator" of English types - PYPWordModel<PoissonUniformWordModel>* up0; // model English lexicon - ConditionalPYPWordModel<PYPWordModel<PoissonUniformWordModel>, FreqBinner>* tmodel; // translation distributions - // (model English word | French word) - const WordID kX; -}; - -#endif diff --git a/gi/pf/itg.cc b/gi/pf/itg.cc deleted file mode 100644 index 29ec3860..00000000 --- a/gi/pf/itg.cc +++ /dev/null @@ -1,275 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -ostream& operator<<(ostream& os, const vector<WordID>& p) { - os << '['; - for (int i = 0; i < p.size(); ++i) - os << (i==0 ? "" : " ") << TD::Convert(p[i]); - return os << ']'; -} - -struct UnigramModel { - explicit UnigramModel(const string& fname, unsigned vocab_size, double p0null = 0.05) : - use_uniform_(fname.size() == 0), - p0null_(p0null), - uniform_((1.0 - p0null) / vocab_size), - probs_(TD::NumWords() + 1) { - if (fname.size() > 0) LoadUnigrams(fname); - probs_[0] = p0null_; - } - -// -// \data\ -// ngram 1=9295 -// -// \1-grams: -// -3.191193 " - - void LoadUnigrams(const string& fname) { - cerr << "Loading unigram probabilities from " << fname << " ..." << endl; - ReadFile rf(fname); - string line; - istream& in = *rf.stream(); - assert(in); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\data\\"); - getline(in, line); - size_t pos = line.find("ngram 1="); - assert(pos == 0); - assert(line.size() > 8); - const size_t num_unigrams = atoi(&line[8]); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\1-grams:"); - for (size_t i = 0; i < num_unigrams; ++i) { - getline(in, line); - assert(line.size() > 0); - pos = line.find('\t'); - assert(pos > 0); - assert(pos + 1 < line.size()); - const WordID w = TD::Convert(line.substr(pos + 1)); - line[pos] = 0; - float p = atof(&line[0]); - const prob_t pnon_null(1.0 - p0null_.as_float()); - if (w < probs_.size()) probs_[w].logeq(p * log(10) + log(pnon_null)); else abort(); - } - } - - const prob_t& operator()(const WordID& w) const { - if (!w) return p0null_; - if (use_uniform_) return uniform_; - return probs_[w]; - } - - const bool use_uniform_; - const prob_t p0null_; - const prob_t uniform_; - vector<prob_t> probs_; -}; - -struct Model1 { - explicit Model1(const string& fname) : - kNULL(TD::Convert("<eps>")), - kZERO() { - LoadModel1(fname); - } - - void LoadModel1(const string& fname) { - cerr << "Loading Model 1 parameters from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - while(getline(in, line)) { - ++lc; - int cur = 0; - int start = 0; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - const WordID src = TD::Convert(&line[0]); - ++cur; - start = cur; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - WordID trg = TD::Convert(&line[start]); - const double logprob = strtod(&line[cur + 1], NULL); - if (src >= ttable.size()) ttable.resize(src + 1); - ttable[src][trg].logeq(logprob); - } - cerr << " read " << lc << " parameters.\n"; - } - - // returns prob 0 if src or trg is not found! - const prob_t& operator()(WordID src, WordID trg) const { - if (src == 0) src = kNULL; - if (src < ttable.size()) { - const map<WordID, prob_t>& cpd = ttable[src]; - const map<WordID, prob_t>::const_iterator it = cpd.find(trg); - if (it != cpd.end()) - return it->second; - } - return kZERO; - } - - const WordID kNULL; - const prob_t kZERO; - vector<map<WordID, prob_t> > ttable; -}; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("particles,p",po::value<unsigned>()->default_value(25),"Number of particles") - ("input,i",po::value<string>(),"Read parallel data from") - ("model1,m",po::value<string>(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value<string>(),"Inverse Model 1 parameters (used in backward estimate)") - ("model1_interpolation_weight",po::value<double>()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("src_unigram,u",po::value<string>()->default_value(""),"Source unigram distribution; empty for uniform") - ("trg_unigram,U",po::value<string>()->default_value(""),"Target unigram distribution; empty for uniform") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector<vector<WordID> >* f, - vector<vector<WordID> >* e, - set<WordID>* vocab_f, - set<WordID>* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector<WordID> tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector<int>()); - f->push_back(vector<int>()); - vector<int>& le = e->back(); - vector<int>& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned particles = conf["particles"].as<unsigned>(); - const unsigned samples = conf["samples"].as<unsigned>(); - TD::Convert("<s>"); - TD::Convert("</s>"); - TD::Convert("<unk>"); - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - boost::shared_ptr<MT19937> prng; - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector<vector<WordID> > corpuse, corpusf; - set<WordID> vocabe, vocabf; - cerr << "Reading corpus...\n"; - ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - UnigramModel src_unigram(conf["src_unigram"].as<string>(), vocabf.size()); - UnigramModel trg_unigram(conf["trg_unigram"].as<string>(), vocabe.size()); - const prob_t kHALF(0.5); - - const string kEMPTY = "NULL"; - const int kLHS = -TD::Convert("X"); - Model1 m1(conf["model1"].as<string>()); - Model1 invm1(conf["inverse_model1"].as<string>()); - for (int si = 0; si < conf["samples"].as<unsigned>(); ++si) { - cerr << '.' << flush; - for (int ci = 0; ci < corpusf.size(); ++ci) { - const vector<WordID>& trg = corpuse[ci]; - const vector<WordID>& src = corpusf[ci]; - for (int i = 0; i <= trg.size(); ++i) { - const WordID e_i = i > 0 ? trg[i-1] : 0; - for (int j = 0; j <= src.size(); ++j) { - const WordID f_j = j > 0 ? src[j-1] : 0; - if (e_i == 0 && f_j == 0) continue; - prob_t je = kHALF * src_unigram(f_j) * m1(f_j,e_i) + kHALF * trg_unigram(e_i) * invm1(e_i,f_j); - cerr << "p( " << (e_i ? TD::Convert(e_i) : kEMPTY) << " , " << (f_j ? TD::Convert(f_j) : kEMPTY) << " ) = " << log(je) << endl; - if (e_i && f_j) - cout << "[X] ||| " << TD::Convert(f_j) << " ||| " << TD::Convert(e_i) << " ||| LogProb=" << log(je) << endl; - } - } - } - } -} - diff --git a/gi/pf/learn_cfg.cc b/gi/pf/learn_cfg.cc deleted file mode 100644 index 1d5126e4..00000000 --- a/gi/pf/learn_cfg.cc +++ /dev/null @@ -1,428 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "inside_outside.h" -#include "hg.h" -#include "bottom_up_parser.h" -#include "fdict.h" -#include "grammar.h" -#include "m.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr<MT19937> prng; -vector<int> nt_vocab; -vector<int> nt_id_to_index; -static unsigned kMAX_RULE_SIZE = 0; -static unsigned kMAX_ARITY = 0; -static bool kALLOW_MIXED = true; // allow rules with mixed terminals and NTs -static bool kHIERARCHICAL_PRIOR = false; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("input,i",po::value<string>(),"Read parallel data from") - ("max_rule_size,m", po::value<unsigned>()->default_value(0), "Maximum rule size (0 for unlimited)") - ("max_arity,a", po::value<unsigned>()->default_value(0), "Maximum number of nonterminals in a rule (0 for unlimited)") - ("no_mixed_rules,M", "Do not mix terminals and nonterminals in a rule RHS") - ("nonterminals,n", po::value<unsigned>()->default_value(1), "Size of nonterminal vocabulary") - ("hierarchical_prior,h", "Use hierarchical prior") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -unsigned ReadCorpus(const string& filename, - vector<vector<WordID> >* e, - set<WordID>* vocab_e) { - e->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - unsigned toks = 0; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector<int>()); - vector<int>& le = e->back(); - TD::ConvertSentence(line, &le); - for (unsigned i = 0; i < le.size(); ++i) - vocab_e->insert(le[i]); - toks += le.size(); - } - if (in != &cin) delete in; - return toks; -} - -struct Grid { - // a b c d e - // 0 - 0 - - - vector<int> grid; -}; - -struct BaseRuleModel { - explicit BaseRuleModel(unsigned term_size, - unsigned nonterm_size = 1) : - unif_term(1.0 / term_size), - unif_nonterm(1.0 / nonterm_size) {} - prob_t operator()(const TRule& r) const { - prob_t p; p.logeq(Md::log_poisson(1.0, r.f_.size())); - const prob_t term_prob((2.0 + 0.01*r.f_.size()) / (r.f_.size() + 2)); - const prob_t nonterm_prob(1.0 - term_prob.as_float()); - for (unsigned i = 0; i < r.f_.size(); ++i) { - if (r.f_[i] <= 0) { // nonterminal - if (kALLOW_MIXED) p *= nonterm_prob; - p *= unif_nonterm; - } else { // terminal - if (kALLOW_MIXED) p *= term_prob; - p *= unif_term; - } - } - return p; - } - const prob_t unif_term, unif_nonterm; -}; - -struct HieroLMModel { - explicit HieroLMModel(unsigned vocab_size, unsigned num_nts = 1) : - base(vocab_size, num_nts), - q0(1,1,1,1), - nts(num_nts, CCRP<TRule>(1,1,1,1)) {} - - prob_t Prob(const TRule& r) const { - return nts[nt_id_to_index[-r.lhs_]].prob(r, p0(r)); - } - - inline prob_t p0(const TRule& r) const { - if (kHIERARCHICAL_PRIOR) - return q0.prob(r, base(r)); - else - return base(r); - } - - int Increment(const TRule& r, MT19937* rng) { - const int delta = nts[nt_id_to_index[-r.lhs_]].increment(r, p0(r), rng); - if (kHIERARCHICAL_PRIOR && delta) - q0.increment(r, base(r), rng); - return delta; - // return x.increment(r); - } - - int Decrement(const TRule& r, MT19937* rng) { - const int delta = nts[nt_id_to_index[-r.lhs_]].decrement(r, rng); - if (kHIERARCHICAL_PRIOR && delta) - q0.decrement(r, rng); - return delta; - //return x.decrement(r); - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (unsigned i = 0; i < nts.size(); ++i) { - prob_t q; q.logeq(nts[i].log_crp_prob()); - p *= q; - for (CCRP<TRule>::const_iterator it = nts[i].begin(); it != nts[i].end(); ++it) { - prob_t tp = p0(it->first); - tp.poweq(it->second.num_tables()); - p *= tp; - } - } - if (kHIERARCHICAL_PRIOR) { - prob_t q; q.logeq(q0.log_crp_prob()); - p *= q; - for (CCRP<TRule>::const_iterator it = q0.begin(); it != q0.end(); ++it) { - prob_t tp = base(it->first); - tp.poweq(it->second.num_tables()); - p *= tp; - } - } - //for (CCRP_OneTable<TRule>::const_iterator it = x.begin(); it != x.end(); ++it) - // p *= base(it->first); - return p; - } - - void ResampleHyperparameters(MT19937* rng) { - for (unsigned i = 0; i < nts.size(); ++i) - nts[i].resample_hyperparameters(rng); - if (kHIERARCHICAL_PRIOR) { - q0.resample_hyperparameters(rng); - cerr << "[base d=" << q0.discount() << ", s=" << q0.strength() << "]"; - } - cerr << " d=" << nts[0].discount() << ", s=" << nts[0].strength() << endl; - } - - const BaseRuleModel base; - CCRP<TRule> q0; - vector<CCRP<TRule> > nts; - //CCRP_OneTable<TRule> x; -}; - -vector<GrammarIter* > tofreelist; - -HieroLMModel* plm; - -struct NPGrammarIter : public GrammarIter, public RuleBin { - NPGrammarIter() : arity() { tofreelist.push_back(this); } - NPGrammarIter(const TRulePtr& inr, const int a, int symbol) : arity(a) { - if (inr) { - r.reset(new TRule(*inr)); - } else { - r.reset(new TRule); - } - TRule& rr = *r; - rr.lhs_ = nt_vocab[0]; - rr.f_.push_back(symbol); - rr.e_.push_back(symbol < 0 ? (1-int(arity)) : symbol); - tofreelist.push_back(this); - } - inline static unsigned NextArity(int cur_a, int symbol) { - return cur_a + (symbol <= 0 ? 1 : 0); - } - virtual int GetNumRules() const { - if (r) return nt_vocab.size(); else return 0; - } - virtual TRulePtr GetIthRule(int i) const { - if (i == 0) return r; - TRulePtr nr(new TRule(*r)); - nr->lhs_ = nt_vocab[i]; - return nr; - } - virtual int Arity() const { - return arity; - } - virtual const RuleBin* GetRules() const { - if (!r) return NULL; else return this; - } - virtual const GrammarIter* Extend(int symbol) const { - const int next_arity = NextArity(arity, symbol); - if (kMAX_ARITY && next_arity > kMAX_ARITY) - return NULL; - if (!kALLOW_MIXED && r) { - bool t1 = r->f_.front() <= 0; - bool t2 = symbol <= 0; - if (t1 != t2) return NULL; - } - if (!kMAX_RULE_SIZE || !r || (r->f_.size() < kMAX_RULE_SIZE)) - return new NPGrammarIter(r, next_arity, symbol); - else - return NULL; - } - const unsigned char arity; - TRulePtr r; -}; - -struct NPGrammar : public Grammar { - virtual const GrammarIter* GetRoot() const { - return new NPGrammarIter; - } -}; - -prob_t TotalProb(const Hypergraph& hg) { - return Inside<prob_t, EdgeProb>(hg); -} - -void SampleDerivation(const Hypergraph& hg, MT19937* rng, vector<unsigned>* sampled_deriv) { - vector<prob_t> node_probs; - Inside<prob_t, EdgeProb>(hg, &node_probs); - queue<unsigned> q; - q.push(hg.nodes_.size() - 2); - while(!q.empty()) { - unsigned cur_node_id = q.front(); -// cerr << "NODE=" << cur_node_id << endl; - q.pop(); - const Hypergraph::Node& node = hg.nodes_[cur_node_id]; - const unsigned num_in_edges = node.in_edges_.size(); - unsigned sampled_edge = 0; - if (num_in_edges == 1) { - sampled_edge = node.in_edges_[0]; - } else { - //prob_t z; - assert(num_in_edges > 1); - SampleSet<prob_t> ss; - for (unsigned j = 0; j < num_in_edges; ++j) { - const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; - prob_t p = edge.edge_prob_; - for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) - p *= node_probs[edge.tail_nodes_[k]]; - ss.add(p); -// cerr << log(ss[j]) << " ||| " << edge.rule_->AsString() << endl; - //z += p; - } -// for (unsigned j = 0; j < num_in_edges; ++j) { -// const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]]; -// cerr << exp(log(ss[j] / z)) << " ||| " << edge.rule_->AsString() << endl; -// } -// cerr << " --- \n"; - sampled_edge = node.in_edges_[rng->SelectSample(ss)]; - } - sampled_deriv->push_back(sampled_edge); - const Hypergraph::Edge& edge = hg.edges_[sampled_edge]; - for (unsigned j = 0; j < edge.tail_nodes_.size(); ++j) { - q.push(edge.tail_nodes_[j]); - } - } - for (unsigned i = 0; i < sampled_deriv->size(); ++i) { - cerr << *hg.edges_[(*sampled_deriv)[i]].rule_ << endl; - } -} - -void IncrementDerivation(const Hypergraph& hg, const vector<unsigned>& d, HieroLMModel* plm, MT19937* rng) { - for (unsigned i = 0; i < d.size(); ++i) - plm->Increment(*hg.edges_[d[i]].rule_, rng); -} - -void DecrementDerivation(const Hypergraph& hg, const vector<unsigned>& d, HieroLMModel* plm, MT19937* rng) { - for (unsigned i = 0; i < d.size(); ++i) - plm->Decrement(*hg.edges_[d[i]].rule_, rng); -} - -int main(int argc, char** argv) { - po::variables_map conf; - - InitCommandLine(argc, argv, &conf); - nt_vocab.resize(conf["nonterminals"].as<unsigned>()); - assert(nt_vocab.size() > 0); - assert(nt_vocab.size() < 26); - { - string nt = "X"; - for (unsigned i = 0; i < nt_vocab.size(); ++i) { - if (nt_vocab.size() > 1) nt[0] = ('A' + i); - int pid = TD::Convert(nt); - nt_vocab[i] = -pid; - if (pid >= nt_id_to_index.size()) { - nt_id_to_index.resize(pid + 1, -1); - } - nt_id_to_index[pid] = i; - } - } - vector<GrammarPtr> grammars; - grammars.push_back(GrammarPtr(new NPGrammar)); - - const unsigned samples = conf["samples"].as<unsigned>(); - kMAX_RULE_SIZE = conf["max_rule_size"].as<unsigned>(); - if (kMAX_RULE_SIZE == 1) { - cerr << "Invalid maximum rule size: must be 0 or >1\n"; - return 1; - } - kMAX_ARITY = conf["max_arity"].as<unsigned>(); - if (kMAX_ARITY == 1) { - cerr << "Invalid maximum arity: must be 0 or >1\n"; - return 1; - } - kALLOW_MIXED = !conf.count("no_mixed_rules"); - - kHIERARCHICAL_PRIOR = conf.count("hierarchical_prior"); - - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - vector<vector<WordID> > corpuse; - set<WordID> vocabe; - cerr << "Reading corpus...\n"; - const unsigned toks = ReadCorpus(conf["input"].as<string>(), &corpuse, &vocabe); - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - HieroLMModel lm(vocabe.size(), nt_vocab.size()); - - plm = &lm; - ExhaustiveBottomUpParser parser(TD::Convert(-nt_vocab[0]), grammars); - - Hypergraph hg; - const int kGoal = -TD::Convert("Goal"); - const int kLP = FD::Convert("LogProb"); - SparseVector<double> v; v.set_value(kLP, 1.0); - vector<vector<unsigned> > derivs(corpuse.size()); - vector<Lattice> cl(corpuse.size()); - for (int ci = 0; ci < corpuse.size(); ++ci) { - vector<int>& src = corpuse[ci]; - Lattice& lat = cl[ci]; - lat.resize(src.size()); - for (unsigned i = 0; i < src.size(); ++i) - lat[i].push_back(LatticeArc(src[i], 0.0, 1)); - } - for (int SS=0; SS < samples; ++SS) { - const bool is_last = ((samples - 1) == SS); - prob_t dlh = prob_t::One(); - for (int ci = 0; ci < corpuse.size(); ++ci) { - const vector<int>& src = corpuse[ci]; - const Lattice& lat = cl[ci]; - cerr << TD::GetString(src) << endl; - hg.clear(); - parser.Parse(lat, &hg); // exhaustive parse - vector<unsigned>& d = derivs[ci]; - if (!is_last) DecrementDerivation(hg, d, &lm, &rng); - for (unsigned i = 0; i < hg.edges_.size(); ++i) { - TRule& r = *hg.edges_[i].rule_; - if (r.lhs_ == kGoal) - hg.edges_[i].edge_prob_ = prob_t::One(); - else - hg.edges_[i].edge_prob_ = lm.Prob(r); - } - if (!is_last) { - d.clear(); - SampleDerivation(hg, &rng, &d); - IncrementDerivation(hg, derivs[ci], &lm, &rng); - } else { - prob_t p = TotalProb(hg); - dlh *= p; - cerr << " p(sentence) = " << log(p) << "\t" << log(dlh) << endl; - } - if (tofreelist.size() > 200000) { - cerr << "Freeing ... "; - for (unsigned i = 0; i < tofreelist.size(); ++i) - delete tofreelist[i]; - tofreelist.clear(); - cerr << "Freed.\n"; - } - } - double llh = log(lm.Likelihood()); - cerr << "LLH=" << llh << "\tENTROPY=" << (-llh / log(2) / toks) << "\tPPL=" << pow(2, -llh / log(2) / toks) << endl; - if (SS % 10 == 9) lm.ResampleHyperparameters(&rng); - if (is_last) { - double z = log(dlh); - cerr << "TOTAL_PROB=" << z << "\tENTROPY=" << (-z / log(2) / toks) << "\tPPL=" << pow(2, -z / log(2) / toks) << endl; - } - } - for (unsigned i = 0; i < nt_vocab.size(); ++i) - cerr << lm.nts[i] << endl; - return 0; -} - diff --git a/gi/pf/make-freq-bins.pl b/gi/pf/make-freq-bins.pl deleted file mode 100755 index fdcd3555..00000000 --- a/gi/pf/make-freq-bins.pl +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $BASE = 6; -my $CUTOFF = 3; - -my %d; -my $num = 0; -while(<>){ - chomp; - my @words = split /\s+/; - for my $w (@words) {$d{$w}++; $num++;} -} - -my @vocab = sort {$d{$b} <=> $d{$a}} keys %d; - -for (my $i=0; $i<scalar @vocab; $i++) { - my $most = $d{$vocab[$i]}; - my $least = 1; - - my $nl = -int(log($most / $num) / log($BASE) + $CUTOFF); - if ($nl < 0) { $nl = 0; } - print "$vocab[$i] $nl\n" -} - - diff --git a/gi/pf/mh_test.cc b/gi/pf/mh_test.cc deleted file mode 100644 index 296e7285..00000000 --- a/gi/pf/mh_test.cc +++ /dev/null @@ -1,148 +0,0 @@ -#include "ccrp.h" - -#include <vector> -#include <iostream> - -#include "tdict.h" -#include "transliterations.h" - -using namespace std; - -MT19937 rng; - -static bool verbose = false; - -struct Model { - - Model() : bp(), base(0.2, 0.6) , ccrps(5, CCRP<int>(0.8, 0.5)) {} - - double p0(int x) const { - assert(x > 0); - assert(x < 5); - return 1.0/4.0; - } - - double llh() const { - double lh = bp + base.log_crp_prob(); - for (int ctx = 1; ctx < 5; ++ctx) - lh += ccrps[ctx].log_crp_prob(); - return lh; - } - - double prob(int ctx, int x) const { - assert(ctx > 0 && ctx < 5); - return ccrps[ctx].prob(x, base.prob(x, p0(x))); - } - - void increment(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - if (ccrps[ctx].increment(x, base.prob(x, p0(x)), &rng)) { - if (base.increment(x, p0(x), &rng)) { - bp += log(1.0 / 4.0); - } - } - } - - // this is just a biased estimate - double est_base_prob(int x) { - return (x + 1) * x / 40.0; - } - - void increment_is(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - SampleSet<double> ss; - const int PARTICLES = 25; - vector<CCRP<int> > s1s(PARTICLES, CCRP<int>(0.5,0.5)); - vector<CCRP<int> > sbs(PARTICLES, CCRP<int>(0.5,0.5)); - vector<double> sp0s(PARTICLES); - - CCRP<int> s1 = ccrps[ctx]; - CCRP<int> sb = base; - double sp0 = bp; - for (int pp = 0; pp < PARTICLES; ++pp) { - if (pp > 0) { - ccrps[ctx] = s1; - base = sb; - bp = sp0; - } - - double q = 1; - double gamma = 1; - double est_p = est_base_prob(x); - //base.prob(x, p0(x)) + rng.next() * 0.1; - if (ccrps[ctx].increment(x, est_p, &rng, &q)) { - gamma = q * base.prob(x, p0(x)); - q *= est_p; - if (verbose) cerr << "(DP-base draw) "; - double qq = -1; - if (base.increment(x, p0(x), &rng, &qq)) { - if (verbose) cerr << "(G0 draw) "; - bp += log(p0(x)); - qq *= p0(x); - } - } else { gamma = q; } - double w = gamma / q; - if (verbose) - cerr << "gamma=" << gamma << " q=" << q << "\tw=" << w << endl; - ss.add(w); - s1s[pp] = ccrps[ctx]; - sbs[pp] = base; - sp0s[pp] = bp; - } - int ps = rng.SelectSample(ss); - ccrps[ctx] = s1s[ps]; - base = sbs[ps]; - bp = sp0s[ps]; - if (verbose) { - cerr << "SELECTED: " << ps << endl; - static int cc = 0; cc++; if (cc ==10) exit(1); - } - } - - void decrement(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - if (ccrps[ctx].decrement(x, &rng)) { - if (base.decrement(x, &rng)) { - bp -= log(p0(x)); - } - } - } - - double bp; - CCRP<int> base; - vector<CCRP<int> > ccrps; - -}; - -int main(int argc, char** argv) { - if (argc > 1) { verbose = true; } - vector<int> counts(15, 0); - vector<int> tcounts(15, 0); - int points[] = {1,2, 2,2, 3,2, 4,1, 3, 4, 3, 3, 2, 3, 4, 1, 4, 1, 3, 2, 1, 3, 1, 4, 0, 0}; - double tlh = 0; - double tt = 0; - for (int n = 0; n < 1000; ++n) { - if (n % 10 == 0) cerr << '.'; - if ((n+1) % 400 == 0) cerr << " [" << (n+1) << "]\n"; - Model m; - for (int *x = points; *x; x += 2) - m.increment(x[0], x[1]); - - for (int j = 0; j < 24; ++j) { - for (int *x = points; *x; x += 2) { - if (rng.next() < 0.8) { - m.decrement(x[0], x[1]); - m.increment_is(x[0], x[1]); - } - } - } - counts[m.base.num_customers()]++; - tcounts[m.base.num_tables()]++; - tlh += m.llh(); - tt += 1.0; - } - cerr << "mean LLH = " << (tlh / tt) << endl; - for (int i = 0; i < 15; ++i) - cerr << i << ": " << (counts[i] / tt) << "\t" << (tcounts[i] / tt) << endl; -} - diff --git a/gi/pf/monotonic_pseg.h b/gi/pf/monotonic_pseg.h deleted file mode 100644 index 10d171fe..00000000 --- a/gi/pf/monotonic_pseg.h +++ /dev/null @@ -1,89 +0,0 @@ -#ifndef _MONOTONIC_PSEG_H_ -#define _MONOTONIC_PSEG_H_ - -#include <vector> - -#include "prob.h" -#include "ccrp_nt.h" -#include "trule.h" -#include "base_distributions.h" - -template <typename BaseMeasure> -struct MonotonicParallelSegementationModel { - explicit MonotonicParallelSegementationModel(BaseMeasure& rcp0) : - rp0(rcp0), base(prob_t::One()), rules(1,1), stop(1.0) {} - - void DecrementRule(const TRule& rule) { - if (rules.decrement(rule)) - base /= rp0(rule); - } - - void IncrementRule(const TRule& rule) { - if (rules.increment(rule)) - base *= rp0(rule); - } - - void IncrementRulesAndStops(const std::vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - if (rules.size()) IncrementContinue(rules.size() - 1); - IncrementStop(); - } - - void DecrementRulesAndStops(const std::vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - if (rules.size()) { - DecrementContinue(rules.size() - 1); - DecrementStop(); - } - } - - prob_t RuleProbability(const TRule& rule) const { - prob_t p; p.logeq(rules.logprob(rule, log(rp0(rule)))); - return p; - } - - prob_t Likelihood() const { - prob_t p = base; - prob_t q; q.logeq(rules.log_crp_prob()); - p *= q; - q.logeq(stop.log_crp_prob()); - p *= q; - return p; - } - - void IncrementStop() { - stop.increment(true); - } - - void IncrementContinue(int n = 1) { - for (int i = 0; i < n; ++i) - stop.increment(false); - } - - void DecrementStop() { - stop.decrement(true); - } - - void DecrementContinue(int n = 1) { - for (int i = 0; i < n; ++i) - stop.decrement(false); - } - - prob_t StopProbability() const { - return prob_t(stop.prob(true, 0.5)); - } - - prob_t ContinueProbability() const { - return prob_t(stop.prob(false, 0.5)); - } - - const BaseMeasure& rp0; - prob_t base; - CCRP_NoTable<TRule> rules; - CCRP_NoTable<bool> stop; -}; - -#endif - diff --git a/gi/pf/ngram_base.cc b/gi/pf/ngram_base.cc deleted file mode 100644 index 1299f06f..00000000 --- a/gi/pf/ngram_base.cc +++ /dev/null @@ -1,69 +0,0 @@ -#include "ngram_base.h" - -#include "lm/model.hh" -#include "tdict.h" - -using namespace std; - -namespace { -struct GICSVMapper : public lm::EnumerateVocab { - GICSVMapper(vector<lm::WordIndex>* out) : out_(out), kLM_UNKNOWN_TOKEN(0) { out_->clear(); } - void Add(lm::WordIndex index, const StringPiece &str) { - const WordID cdec_id = TD::Convert(str.as_string()); - if (cdec_id >= out_->size()) - out_->resize(cdec_id + 1, kLM_UNKNOWN_TOKEN); - (*out_)[cdec_id] = index; - } - vector<lm::WordIndex>* out_; - const lm::WordIndex kLM_UNKNOWN_TOKEN; -}; -} - -struct FixedNgramBaseImpl { - FixedNgramBaseImpl(const string& param) { - GICSVMapper vm(&cdec2klm_map_); - lm::ngram::Config conf; - conf.enumerate_vocab = &vm; - cerr << "Reading character LM from " << param << endl; - model = new lm::ngram::ProbingModel(param.c_str(), conf); - order = model->Order(); - kEOS = MapWord(TD::Convert("</s>")); - assert(kEOS > 0); - } - - lm::WordIndex MapWord(const WordID w) const { - if (w < cdec2klm_map_.size()) return cdec2klm_map_[w]; - return 0; - } - - ~FixedNgramBaseImpl() { delete model; } - - prob_t StringProbability(const vector<WordID>& s) const { - lm::ngram::State state = model->BeginSentenceState(); - double prob = 0; - for (unsigned i = 0; i < s.size(); ++i) { - const lm::ngram::State scopy(state); - prob += model->Score(scopy, MapWord(s[i]), state); - } - const lm::ngram::State scopy(state); - prob += model->Score(scopy, kEOS, state); - prob_t p; p.logeq(prob * log(10)); - return p; - } - - lm::ngram::ProbingModel* model; - unsigned order; - vector<lm::WordIndex> cdec2klm_map_; - lm::WordIndex kEOS; -}; - -FixedNgramBase::~FixedNgramBase() { delete impl; } - -FixedNgramBase::FixedNgramBase(const string& lmfname) { - impl = new FixedNgramBaseImpl(lmfname); -} - -prob_t FixedNgramBase::StringProbability(const vector<WordID>& s) const { - return impl->StringProbability(s); -} - diff --git a/gi/pf/ngram_base.h b/gi/pf/ngram_base.h deleted file mode 100644 index 4ea999f3..00000000 --- a/gi/pf/ngram_base.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef _NGRAM_BASE_H_ -#define _NGRAM_BASE_H_ - -#include <string> -#include <vector> -#include "trule.h" -#include "wordid.h" -#include "prob.h" - -struct FixedNgramBaseImpl; -struct FixedNgramBase { - FixedNgramBase(const std::string& lmfname); - ~FixedNgramBase(); - prob_t StringProbability(const std::vector<WordID>& s) const; - - prob_t operator()(const TRule& rule) const { - return StringProbability(rule.e_); - } - - private: - FixedNgramBaseImpl* impl; - -}; - -#endif diff --git a/gi/pf/nuisance_test.cc b/gi/pf/nuisance_test.cc deleted file mode 100644 index fc0af9cb..00000000 --- a/gi/pf/nuisance_test.cc +++ /dev/null @@ -1,161 +0,0 @@ -#include "ccrp.h" - -#include <vector> -#include <iostream> - -#include "tdict.h" -#include "transliterations.h" - -using namespace std; - -MT19937 rng; - -ostream& operator<<(ostream&os, const vector<int>& v) { - os << '[' << v[0]; - if (v.size() == 2) os << ' ' << v[1]; - return os << ']'; -} - -struct Base { - Base() : llh(), v(2), v1(1), v2(1), crp(0.25, 0.5) {} - inline double p0(const vector<int>& x) const { - double p = 0.75; - if (x.size() == 2) p = 0.25; - p *= 1.0 / 3.0; - if (x.size() == 2) p *= 1.0 / 3.0; - return p; - } - double est_deriv_prob(int a, int b, int seg) const { - assert(a > 0 && a < 4); // a \in {1,2,3} - assert(b > 0 && b < 4); // b \in {1,2,3} - assert(seg == 0 || seg == 1); // seg \in {0,1} - if (seg == 0) { - v[0] = a; - v[1] = b; - return crp.prob(v, p0(v)); - } else { - v1[0] = a; - v2[0] = b; - return crp.prob(v1, p0(v1)) * crp.prob(v2, p0(v2)); - } - } - double est_marginal_prob(int a, int b) const { - return est_deriv_prob(a,b,0) + est_deriv_prob(a,b,1); - } - int increment(int a, int b, double* pw = NULL) { - double p1 = est_deriv_prob(a, b, 0); - double p2 = est_deriv_prob(a, b, 1); - //p1 = 0.5; p2 = 0.5; - int seg = rng.SelectSample(p1,p2); - double tmp = 0; - if (!pw) pw = &tmp; - double& w = *pw; - if (seg == 0) { - v[0] = a; - v[1] = b; - w = crp.prob(v, p0(v)) / p1; - if (crp.increment(v, p0(v), &rng)) { - llh += log(p0(v)); - } - } else { - v1[0] = a; - w = crp.prob(v1, p0(v1)) / p2; - if (crp.increment(v1, p0(v1), &rng)) { - llh += log(p0(v1)); - } - v2[0] = b; - w *= crp.prob(v2, p0(v2)); - if (crp.increment(v2, p0(v2), &rng)) { - llh += log(p0(v2)); - } - } - return seg; - } - void increment(int a, int b, int seg) { - if (seg == 0) { - v[0] = a; - v[1] = b; - if (crp.increment(v, p0(v), &rng)) { - llh += log(p0(v)); - } - } else { - v1[0] = a; - if (crp.increment(v1, p0(v1), &rng)) { - llh += log(p0(v1)); - } - v2[0] = b; - if (crp.increment(v2, p0(v2), &rng)) { - llh += log(p0(v2)); - } - } - } - void decrement(int a, int b, int seg) { - if (seg == 0) { - v[0] = a; - v[1] = b; - if (crp.decrement(v, &rng)) { - llh -= log(p0(v)); - } - } else { - v1[0] = a; - if (crp.decrement(v1, &rng)) { - llh -= log(p0(v1)); - } - v2[0] = b; - if (crp.decrement(v2, &rng)) { - llh -= log(p0(v2)); - } - } - } - double log_likelihood() const { - return llh + crp.log_crp_prob(); - } - double llh; - mutable vector<int> v, v1, v2; - CCRP<vector<int> > crp; -}; - -int main(int argc, char** argv) { - double tl = 0; - const int ITERS = 1000; - const int PARTICLES = 20; - const int DATAPOINTS = 50; - WordID x = TD::Convert("souvenons"); - WordID y = TD::Convert("remember"); - vector<WordID> src; TD::ConvertSentence("s o u v e n o n s", &src); - vector<WordID> trg; TD::ConvertSentence("r e m e m b e r", &trg); -// Transliterations xx; -// xx.Initialize(x, src, y, trg); -// return 1; - - for (int j = 0; j < ITERS; ++j) { - Base b; - vector<int> segs(DATAPOINTS); - SampleSet<double> ss; - vector<int> sss; - for (int i = 0; i < DATAPOINTS; i++) { - ss.clear(); - sss.clear(); - int x = ((i / 10) % 3) + 1; - int y = (i % 3) + 1; - //double ep = b.est_marginal_prob(x,y); - //cerr << "est p(" << x << "," << y << ") = " << ep << endl; - for (int n = 0; n < PARTICLES; ++n) { - double w; - int seg = b.increment(x,y,&w); - //cerr << seg << " w=" << w << endl; - ss.add(w); - sss.push_back(seg); - b.decrement(x,y,seg); - } - int seg = sss[rng.SelectSample(ss)]; - b.increment(x, y, seg); - //cerr << "Selected: " << seg << endl; - //return 1; - segs[i] = seg; - } - tl += b.log_likelihood(); - } - cerr << "LLH=" << tl / ITERS << endl; -} - diff --git a/gi/pf/os_phrase.h b/gi/pf/os_phrase.h deleted file mode 100644 index dfe40cb1..00000000 --- a/gi/pf/os_phrase.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef _OS_PHRASE_H_ -#define _OS_PHRASE_H_ - -#include <iostream> -#include <vector> -#include "tdict.h" - -inline std::ostream& operator<<(std::ostream& os, const std::vector<WordID>& p) { - os << '['; - for (int i = 0; i < p.size(); ++i) - os << (i==0 ? "" : " ") << TD::Convert(p[i]); - return os << ']'; -} - -#endif diff --git a/gi/pf/pf.h b/gi/pf/pf.h deleted file mode 100644 index ede7cda8..00000000 --- a/gi/pf/pf.h +++ /dev/null @@ -1,84 +0,0 @@ -#ifndef _PF_H_ -#define _PF_H_ - -#include <cassert> -#include <vector> -#include "sampler.h" -#include "prob.h" - -template <typename ParticleType> -struct ParticleRenormalizer { - void operator()(std::vector<ParticleType>* pv) const { - if (pv->empty()) return; - prob_t z = prob_t::Zero(); - for (unsigned i = 0; i < pv->size(); ++i) - z += (*pv)[i].weight; - assert(z > prob_t::Zero()); - for (unsigned i = 0; i < pv->size(); ++i) - (*pv)[i].weight /= z; - } -}; - -template <typename ParticleType> -struct MultinomialResampleFilter { - explicit MultinomialResampleFilter(MT19937* rng) : rng_(rng) {} - - void operator()(std::vector<ParticleType>* pv) { - if (pv->empty()) return; - std::vector<ParticleType>& ps = *pv; - SampleSet<prob_t> ss; - for (int i = 0; i < ps.size(); ++i) - ss.add(ps[i].weight); - std::vector<ParticleType> nps; nps.reserve(ps.size()); - const prob_t uniform_weight(1.0 / ps.size()); - for (int i = 0; i < ps.size(); ++i) { - nps.push_back(ps[rng_->SelectSample(ss)]); - nps[i].weight = uniform_weight; - } - nps.swap(ps); - } - - private: - MT19937* rng_; -}; - -template <typename ParticleType> -struct SystematicResampleFilter { - explicit SystematicResampleFilter(MT19937* rng) : rng_(rng), renorm_() {} - - void operator()(std::vector<ParticleType>* pv) { - if (pv->empty()) return; - renorm_(pv); - std::vector<ParticleType>& ps = *pv; - std::vector<ParticleType> nps; nps.reserve(ps.size()); - double lower = 0, upper = 0; - const double skip = 1.0 / ps.size(); - double u_j = rng_->next() * skip; - //std::cerr << "u_0: " << u_j << std::endl; - int j = 0; - for (unsigned i = 0; i < ps.size(); ++i) { - upper += ps[i].weight.as_float(); - //std::cerr << "lower: " << lower << " upper: " << upper << std::endl; - // how many children does ps[i] have? - while (u_j < lower) { u_j += skip; ++j; } - while (u_j >= lower && u_j <= upper) { - assert(j < ps.size()); - nps.push_back(ps[i]); - u_j += skip; - //std::cerr << " add u_j=" << u_j << std::endl; - ++j; - } - lower = upper; - } - //std::cerr << ps.size() << " " << nps.size() << "\n"; - assert(ps.size() == nps.size()); - //exit(1); - ps.swap(nps); - } - - private: - MT19937* rng_; - ParticleRenormalizer<ParticleType> renorm_; -}; - -#endif diff --git a/gi/pf/pf_test.cc b/gi/pf/pf_test.cc deleted file mode 100644 index 296e7285..00000000 --- a/gi/pf/pf_test.cc +++ /dev/null @@ -1,148 +0,0 @@ -#include "ccrp.h" - -#include <vector> -#include <iostream> - -#include "tdict.h" -#include "transliterations.h" - -using namespace std; - -MT19937 rng; - -static bool verbose = false; - -struct Model { - - Model() : bp(), base(0.2, 0.6) , ccrps(5, CCRP<int>(0.8, 0.5)) {} - - double p0(int x) const { - assert(x > 0); - assert(x < 5); - return 1.0/4.0; - } - - double llh() const { - double lh = bp + base.log_crp_prob(); - for (int ctx = 1; ctx < 5; ++ctx) - lh += ccrps[ctx].log_crp_prob(); - return lh; - } - - double prob(int ctx, int x) const { - assert(ctx > 0 && ctx < 5); - return ccrps[ctx].prob(x, base.prob(x, p0(x))); - } - - void increment(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - if (ccrps[ctx].increment(x, base.prob(x, p0(x)), &rng)) { - if (base.increment(x, p0(x), &rng)) { - bp += log(1.0 / 4.0); - } - } - } - - // this is just a biased estimate - double est_base_prob(int x) { - return (x + 1) * x / 40.0; - } - - void increment_is(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - SampleSet<double> ss; - const int PARTICLES = 25; - vector<CCRP<int> > s1s(PARTICLES, CCRP<int>(0.5,0.5)); - vector<CCRP<int> > sbs(PARTICLES, CCRP<int>(0.5,0.5)); - vector<double> sp0s(PARTICLES); - - CCRP<int> s1 = ccrps[ctx]; - CCRP<int> sb = base; - double sp0 = bp; - for (int pp = 0; pp < PARTICLES; ++pp) { - if (pp > 0) { - ccrps[ctx] = s1; - base = sb; - bp = sp0; - } - - double q = 1; - double gamma = 1; - double est_p = est_base_prob(x); - //base.prob(x, p0(x)) + rng.next() * 0.1; - if (ccrps[ctx].increment(x, est_p, &rng, &q)) { - gamma = q * base.prob(x, p0(x)); - q *= est_p; - if (verbose) cerr << "(DP-base draw) "; - double qq = -1; - if (base.increment(x, p0(x), &rng, &qq)) { - if (verbose) cerr << "(G0 draw) "; - bp += log(p0(x)); - qq *= p0(x); - } - } else { gamma = q; } - double w = gamma / q; - if (verbose) - cerr << "gamma=" << gamma << " q=" << q << "\tw=" << w << endl; - ss.add(w); - s1s[pp] = ccrps[ctx]; - sbs[pp] = base; - sp0s[pp] = bp; - } - int ps = rng.SelectSample(ss); - ccrps[ctx] = s1s[ps]; - base = sbs[ps]; - bp = sp0s[ps]; - if (verbose) { - cerr << "SELECTED: " << ps << endl; - static int cc = 0; cc++; if (cc ==10) exit(1); - } - } - - void decrement(int ctx, int x) { - assert(ctx > 0 && ctx < 5); - if (ccrps[ctx].decrement(x, &rng)) { - if (base.decrement(x, &rng)) { - bp -= log(p0(x)); - } - } - } - - double bp; - CCRP<int> base; - vector<CCRP<int> > ccrps; - -}; - -int main(int argc, char** argv) { - if (argc > 1) { verbose = true; } - vector<int> counts(15, 0); - vector<int> tcounts(15, 0); - int points[] = {1,2, 2,2, 3,2, 4,1, 3, 4, 3, 3, 2, 3, 4, 1, 4, 1, 3, 2, 1, 3, 1, 4, 0, 0}; - double tlh = 0; - double tt = 0; - for (int n = 0; n < 1000; ++n) { - if (n % 10 == 0) cerr << '.'; - if ((n+1) % 400 == 0) cerr << " [" << (n+1) << "]\n"; - Model m; - for (int *x = points; *x; x += 2) - m.increment(x[0], x[1]); - - for (int j = 0; j < 24; ++j) { - for (int *x = points; *x; x += 2) { - if (rng.next() < 0.8) { - m.decrement(x[0], x[1]); - m.increment_is(x[0], x[1]); - } - } - } - counts[m.base.num_customers()]++; - tcounts[m.base.num_tables()]++; - tlh += m.llh(); - tt += 1.0; - } - cerr << "mean LLH = " << (tlh / tt) << endl; - for (int i = 0; i < 15; ++i) - cerr << i << ": " << (counts[i] / tt) << "\t" << (tcounts[i] / tt) << endl; -} - diff --git a/gi/pf/pfbrat.cc b/gi/pf/pfbrat.cc deleted file mode 100644 index 832f22cf..00000000 --- a/gi/pf/pfbrat.cc +++ /dev/null @@ -1,543 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/multi_array.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "cfg_wfst_composer.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -static unsigned kMAX_SRC_PHRASE; -static unsigned kMAX_TRG_PHRASE; -struct FSTState; - -double log_poisson(unsigned x, const double& lambda) { - assert(lambda > 0.0); - return log(lambda) * x - lgamma(x + 1) - lambda; -} - -struct ConditionalBase { - explicit ConditionalBase(const double m1mixture, const unsigned vocab_e_size, const string& model1fname) : - kM1MIXTURE(m1mixture), - kUNIFORM_MIXTURE(1.0 - m1mixture), - kUNIFORM_TARGET(1.0 / vocab_e_size), - kNULL(TD::Convert("<eps>")) { - assert(m1mixture >= 0.0 && m1mixture <= 1.0); - assert(vocab_e_size > 0); - LoadModel1(model1fname); - } - - void LoadModel1(const string& fname) { - cerr << "Loading Model 1 parameters from " << fname << " ..." << endl; - ReadFile rf(fname); - istream& in = *rf.stream(); - string line; - unsigned lc = 0; - while(getline(in, line)) { - ++lc; - int cur = 0; - int start = 0; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - const WordID src = TD::Convert(&line[0]); - ++cur; - start = cur; - while(cur < line.size() && line[cur] != ' ') { ++cur; } - assert(cur != line.size()); - line[cur] = 0; - WordID trg = TD::Convert(&line[start]); - const double logprob = strtod(&line[cur + 1], NULL); - if (src >= ttable.size()) ttable.resize(src + 1); - ttable[src][trg].logeq(logprob); - } - cerr << " read " << lc << " parameters.\n"; - } - - // return logp0 of rule.e_ | rule.f_ - prob_t operator()(const TRule& rule) const { - const int flen = rule.f_.size(); - const int elen = rule.e_.size(); - prob_t uniform_src_alignment; uniform_src_alignment.logeq(-log(flen + 1)); - prob_t p; - p.logeq(log_poisson(elen, flen + 0.01)); // elen | flen ~Pois(flen + 0.01) - for (int i = 0; i < elen; ++i) { // for each position i in e-RHS - const WordID trg = rule.e_[i]; - prob_t tp = prob_t::Zero(); - for (int j = -1; j < flen; ++j) { - const WordID src = j < 0 ? kNULL : rule.f_[j]; - const map<WordID, prob_t>::const_iterator it = ttable[src].find(trg); - if (it != ttable[src].end()) { - tp += kM1MIXTURE * it->second; - } - tp += kUNIFORM_MIXTURE * kUNIFORM_TARGET; - } - tp *= uniform_src_alignment; // draw a_i ~uniform - p *= tp; // draw e_i ~Model1(f_a_i) / uniform - } - return p; - } - - const prob_t kM1MIXTURE; // Model 1 mixture component - const prob_t kUNIFORM_MIXTURE; // uniform mixture component - const prob_t kUNIFORM_TARGET; - const WordID kNULL; - vector<map<WordID, prob_t> > ttable; -}; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("input,i",po::value<string>(),"Read parallel data from") - ("max_src_phrase",po::value<unsigned>()->default_value(3),"Maximum length of source language phrases") - ("max_trg_phrase",po::value<unsigned>()->default_value(3),"Maximum length of target language phrases") - ("model1,m",po::value<string>(),"Model 1 parameters (used in base distribution)") - ("model1_interpolation_weight",po::value<double>()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector<vector<WordID> >* f, - vector<vector<int> >* e, - set<int>* vocab_f, - set<int>* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector<WordID> tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector<int>()); - f->push_back(vector<int>()); - vector<int>& le = e->back(); - vector<int>& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -struct UniphraseLM { - UniphraseLM(const vector<vector<int> >& corpus, - const set<int>& vocab, - const po::variables_map& conf) : - phrases_(1,1), - gen_(1,1), - corpus_(corpus), - uniform_word_(1.0 / vocab.size()), - gen_p0_(0.5), - p_end_(0.5), - use_poisson_(conf.count("poisson_length") > 0) {} - - void ResampleHyperparameters(MT19937* rng) { - phrases_.resample_hyperparameters(rng); - gen_.resample_hyperparameters(rng); - cerr << " " << phrases_.alpha(); - } - - CCRP_NoTable<vector<int> > phrases_; - CCRP_NoTable<bool> gen_; - vector<vector<bool> > z_; // z_[i] is there a phrase boundary after the ith word - const vector<vector<int> >& corpus_; - const double uniform_word_; - const double gen_p0_; - const double p_end_; // in base length distribution, p of the end of a phrase - const bool use_poisson_; -}; - -struct Reachability { - boost::multi_array<bool, 4> edges; // edges[src_covered][trg_covered][x][trg_delta] is this edge worth exploring? - boost::multi_array<short, 2> max_src_delta; // msd[src_covered][trg_covered] -- the largest src delta that's valid - - Reachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) : - edges(boost::extents[srclen][trglen][src_max_phrase_len+1][trg_max_phrase_len+1]), - max_src_delta(boost::extents[srclen][trglen]) { - ComputeReachability(srclen, trglen, src_max_phrase_len, trg_max_phrase_len); - } - - private: - struct SState { - SState() : prev_src_covered(), prev_trg_covered() {} - SState(int i, int j) : prev_src_covered(i), prev_trg_covered(j) {} - int prev_src_covered; - int prev_trg_covered; - }; - - struct NState { - NState() : next_src_covered(), next_trg_covered() {} - NState(int i, int j) : next_src_covered(i), next_trg_covered(j) {} - int next_src_covered; - int next_trg_covered; - }; - - void ComputeReachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) { - typedef boost::multi_array<vector<SState>, 2> array_type; - array_type a(boost::extents[srclen + 1][trglen + 1]); - a[0][0].push_back(SState()); - for (int i = 0; i < srclen; ++i) { - for (int j = 0; j < trglen; ++j) { - if (a[i][j].size() == 0) continue; - const SState prev(i,j); - for (int k = 1; k <= src_max_phrase_len; ++k) { - if ((i + k) > srclen) continue; - for (int l = 1; l <= trg_max_phrase_len; ++l) { - if ((j + l) > trglen) continue; - a[i + k][j + l].push_back(prev); - } - } - } - } - a[0][0].clear(); - cerr << "Final cell contains " << a[srclen][trglen].size() << " back pointers\n"; - assert(a[srclen][trglen].size() > 0); - - typedef boost::multi_array<bool, 2> rarray_type; - rarray_type r(boost::extents[srclen + 1][trglen + 1]); -// typedef boost::multi_array<vector<NState>, 2> narray_type; -// narray_type b(boost::extents[srclen + 1][trglen + 1]); - r[srclen][trglen] = true; - for (int i = srclen; i >= 0; --i) { - for (int j = trglen; j >= 0; --j) { - vector<SState>& prevs = a[i][j]; - if (!r[i][j]) { prevs.clear(); } -// const NState nstate(i,j); - for (int k = 0; k < prevs.size(); ++k) { - r[prevs[k].prev_src_covered][prevs[k].prev_trg_covered] = true; - int src_delta = i - prevs[k].prev_src_covered; - edges[prevs[k].prev_src_covered][prevs[k].prev_trg_covered][src_delta][j - prevs[k].prev_trg_covered] = true; - short &msd = max_src_delta[prevs[k].prev_src_covered][prevs[k].prev_trg_covered]; - if (src_delta > msd) msd = src_delta; -// b[prevs[k].prev_src_covered][prevs[k].prev_trg_covered].push_back(nstate); - } - } - } - assert(!edges[0][0][1][0]); - assert(!edges[0][0][0][1]); - assert(!edges[0][0][0][0]); - cerr << " MAX SRC DELTA[0][0] = " << max_src_delta[0][0] << endl; - assert(max_src_delta[0][0] > 0); - //cerr << "First cell contains " << b[0][0].size() << " forward pointers\n"; - //for (int i = 0; i < b[0][0].size(); ++i) { - // cerr << " -> (" << b[0][0][i].next_src_covered << "," << b[0][0][i].next_trg_covered << ")\n"; - //} - } -}; - -ostream& operator<<(ostream& os, const FSTState& q); -struct FSTState { - explicit FSTState(int src_size) : - trg_covered_(), - src_covered_(), - src_coverage_(src_size) {} - - FSTState(short trg_covered, short src_covered, const vector<bool>& src_coverage, const vector<short>& src_prefix) : - trg_covered_(trg_covered), - src_covered_(src_covered), - src_coverage_(src_coverage), - src_prefix_(src_prefix) { - if (src_coverage_.size() == src_covered) { - assert(src_prefix.size() == 0); - } - } - - // if we extend by the word at src_position, what are - // the next states that are reachable and lie on a valid - // path to the final state? - vector<FSTState> Extensions(int src_position, int src_len, int trg_len, const Reachability& r) const { - assert(src_position < src_coverage_.size()); - if (src_coverage_[src_position]) { - cerr << "Trying to extend " << *this << " with position " << src_position << endl; - abort(); - } - vector<bool> ncvg = src_coverage_; - ncvg[src_position] = true; - - vector<FSTState> res; - const int trg_remaining = trg_len - trg_covered_; - if (trg_remaining <= 0) { - cerr << "Target appears to have been covered: " << *this << " (trg_len=" << trg_len << ",trg_covered=" << trg_covered_ << ")" << endl; - abort(); - } - const int src_remaining = src_len - src_covered_; - if (src_remaining <= 0) { - cerr << "Source appears to have been covered: " << *this << endl; - abort(); - } - - for (int tc = 1; tc <= kMAX_TRG_PHRASE; ++tc) { - if (r.edges[src_covered_][trg_covered_][src_prefix_.size() + 1][tc]) { - int nc = src_prefix_.size() + 1 + src_covered_; - res.push_back(FSTState(trg_covered_ + tc, nc, ncvg, vector<short>())); - } - } - - if ((src_prefix_.size() + 1) < r.max_src_delta[src_covered_][trg_covered_]) { - vector<short> nsp = src_prefix_; - nsp.push_back(src_position); - res.push_back(FSTState(trg_covered_, src_covered_, ncvg, nsp)); - } - - if (res.size() == 0) { - cerr << *this << " can't be extended!\n"; - abort(); - } - return res; - } - - short trg_covered_, src_covered_; - vector<bool> src_coverage_; - vector<short> src_prefix_; -}; -bool operator<(const FSTState& q, const FSTState& r) { - if (q.trg_covered_ != r.trg_covered_) return q.trg_covered_ < r.trg_covered_; - if (q.src_covered_!= r.src_covered_) return q.src_covered_ < r.src_covered_; - if (q.src_coverage_ != r.src_coverage_) return q.src_coverage_ < r.src_coverage_; - return q.src_prefix_ < r.src_prefix_; -} - -ostream& operator<<(ostream& os, const FSTState& q) { - os << "[" << q.trg_covered_ << " : "; - for (int i = 0; i < q.src_coverage_.size(); ++i) - os << q.src_coverage_[i]; - os << " : <"; - for (int i = 0; i < q.src_prefix_.size(); ++i) { - if (i != 0) os << ' '; - os << q.src_prefix_[i]; - } - return os << ">]"; -} - -struct MyModel { - MyModel(ConditionalBase& rcp0) : rp0(rcp0) {} - typedef unordered_map<vector<WordID>, CCRP_NoTable<TRule>, boost::hash<vector<WordID> > > SrcToRuleCRPMap; - - void DecrementRule(const TRule& rule) { - SrcToRuleCRPMap::iterator it = rules.find(rule.f_); - assert(it != rules.end()); - it->second.decrement(rule); - if (it->second.num_customers() == 0) rules.erase(it); - } - - void IncrementRule(const TRule& rule) { - SrcToRuleCRPMap::iterator it = rules.find(rule.f_); - if (it == rules.end()) { - CCRP_NoTable<TRule> crp(1,1); - it = rules.insert(make_pair(rule.f_, crp)).first; - } - it->second.increment(rule); - } - - // conditioned on rule.f_ - prob_t RuleConditionalProbability(const TRule& rule) const { - const prob_t base = rp0(rule); - SrcToRuleCRPMap::const_iterator it = rules.find(rule.f_); - if (it == rules.end()) { - return base; - } else { - const double lp = it->second.logprob(rule, log(base)); - prob_t q; q.logeq(lp); - return q; - } - } - - const ConditionalBase& rp0; - SrcToRuleCRPMap rules; -}; - -struct MyFST : public WFST { - MyFST(const vector<WordID>& ssrc, const vector<WordID>& strg, MyModel* m) : - src(ssrc), trg(strg), - r(src.size(),trg.size(),kMAX_SRC_PHRASE, kMAX_TRG_PHRASE), - model(m) { - FSTState in(src.size()); - cerr << " INIT: " << in << endl; - init = GetNode(in); - for (int i = 0; i < in.src_coverage_.size(); ++i) in.src_coverage_[i] = true; - in.src_covered_ = src.size(); - in.trg_covered_ = trg.size(); - cerr << "FINAL: " << in << endl; - final = GetNode(in); - } - virtual const WFSTNode* Final() const; - virtual const WFSTNode* Initial() const; - - const WFSTNode* GetNode(const FSTState& q); - map<FSTState, boost::shared_ptr<WFSTNode> > m; - const vector<WordID>& src; - const vector<WordID>& trg; - Reachability r; - const WFSTNode* init; - const WFSTNode* final; - MyModel* model; -}; - -struct MyNode : public WFSTNode { - MyNode(const FSTState& q, MyFST* fst) : state(q), container(fst) {} - virtual vector<pair<const WFSTNode*, TRulePtr> > ExtendInput(unsigned srcindex) const; - const FSTState state; - mutable MyFST* container; -}; - -vector<pair<const WFSTNode*, TRulePtr> > MyNode::ExtendInput(unsigned srcindex) const { - cerr << "EXTEND " << state << " with " << srcindex << endl; - vector<FSTState> ext = state.Extensions(srcindex, container->src.size(), container->trg.size(), container->r); - vector<pair<const WFSTNode*,TRulePtr> > res(ext.size()); - for (unsigned i = 0; i < ext.size(); ++i) { - res[i].first = container->GetNode(ext[i]); - if (ext[i].src_prefix_.size() == 0) { - const unsigned trg_from = state.trg_covered_; - const unsigned trg_to = ext[i].trg_covered_; - const unsigned prev_prfx_size = state.src_prefix_.size(); - res[i].second.reset(new TRule); - res[i].second->lhs_ = -TD::Convert("X"); - vector<WordID>& src = res[i].second->f_; - vector<WordID>& trg = res[i].second->e_; - src.resize(prev_prfx_size + 1); - for (unsigned j = 0; j < prev_prfx_size; ++j) - src[j] = container->src[state.src_prefix_[j]]; - src[prev_prfx_size] = container->src[srcindex]; - for (unsigned j = trg_from; j < trg_to; ++j) - trg.push_back(container->trg[j]); - res[i].second->scores_.set_value(FD::Convert("Proposal"), log(container->model->RuleConditionalProbability(*res[i].second))); - } - } - return res; -} - -const WFSTNode* MyFST::GetNode(const FSTState& q) { - boost::shared_ptr<WFSTNode>& res = m[q]; - if (!res) { - res.reset(new MyNode(q, this)); - } - return &*res; -} - -const WFSTNode* MyFST::Final() const { - return final; -} - -const WFSTNode* MyFST::Initial() const { - return init; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - kMAX_TRG_PHRASE = conf["max_trg_phrase"].as<unsigned>(); - kMAX_SRC_PHRASE = conf["max_src_phrase"].as<unsigned>(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - boost::shared_ptr<MT19937> prng; - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector<vector<int> > corpuse, corpusf; - set<int> vocabe, vocabf; - ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "f-Corpus size: " << corpusf.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabf.size() << " types\n"; - cerr << "f-Corpus size: " << corpuse.size() << " sentences\n"; - cerr << "f-Vocabulary size: " << vocabe.size() << " types\n"; - assert(corpusf.size() == corpuse.size()); - - ConditionalBase lp0(conf["model1_interpolation_weight"].as<double>(), - vocabe.size(), - conf["model1"].as<string>()); - MyModel m(lp0); - - TRule x("[X] ||| kAnwntR myN ||| at the convent ||| 0"); - m.IncrementRule(x); - TRule y("[X] ||| nY dyN ||| gave ||| 0"); - m.IncrementRule(y); - - - MyFST fst(corpusf[0], corpuse[0], &m); - ifstream in("./kimura.g"); - assert(in); - CFG_WFSTComposer comp(fst); - Hypergraph hg; - bool succeed = comp.Compose(&in, &hg); - hg.PrintGraphviz(); - if (succeed) { cerr << "SUCCESS.\n"; } else { cerr << "FAILURE REPORTED.\n"; } - -#if 0 - ifstream in2("./amnabooks.g"); - assert(in2); - MyFST fst2(corpusf[1], corpuse[1], &m); - CFG_WFSTComposer comp2(fst2); - Hypergraph hg2; - bool succeed2 = comp2.Compose(&in2, &hg2); - if (succeed2) { cerr << "SUCCESS.\n"; } else { cerr << "FAILURE REPORTED.\n"; } -#endif - - SparseVector<double> w; w.set_value(FD::Convert("Proposal"), 1.0); - hg.Reweight(w); - cerr << ViterbiFTree(hg) << endl; - return 0; -} - diff --git a/gi/pf/pfdist.cc b/gi/pf/pfdist.cc deleted file mode 100644 index a3e46064..00000000 --- a/gi/pf/pfdist.cc +++ /dev/null @@ -1,598 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "pf.h" -#include "base_distributions.h" -#include "reachability.h" -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr<MT19937> prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("particles,p",po::value<unsigned>()->default_value(30),"Number of particles") - ("filter_frequency,f",po::value<unsigned>()->default_value(5),"Number of time steps between filterings") - ("input,i",po::value<string>(),"Read parallel data from") - ("max_src_phrase",po::value<unsigned>()->default_value(5),"Maximum length of source language phrases") - ("max_trg_phrase",po::value<unsigned>()->default_value(5),"Maximum length of target language phrases") - ("model1,m",po::value<string>(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value<string>(),"Inverse Model 1 parameters (used in backward estimate)") - ("model1_interpolation_weight",po::value<double>()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector<vector<WordID> >* f, - vector<vector<WordID> >* e, - set<WordID>* vocab_f, - set<WordID>* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector<WordID> tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector<int>()); - f->push_back(vector<int>()); - vector<int>& le = e->back(); - vector<int>& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -#if 0 -struct MyConditionalModel { - MyConditionalModel(PhraseConditionalBase& rcp0) : rp0(&rcp0), base(prob_t::One()), src_phrases(1,1), src_jumps(200, CCRP_NoTable<int>(1,1)) {} - - prob_t srcp0(const vector<WordID>& src) const { - prob_t p(1.0 / 3000.0); - p.poweq(src.size()); - prob_t lenp; lenp.logeq(log_poisson(src.size(), 1.0)); - p *= lenp; - return p; - } - - void DecrementRule(const TRule& rule) { - const RuleCRPMap::iterator it = rules.find(rule.f_); - assert(it != rules.end()); - if (it->second.decrement(rule)) { - base /= (*rp0)(rule); - if (it->second.num_customers() == 0) - rules.erase(it); - } - if (src_phrases.decrement(rule.f_)) - base /= srcp0(rule.f_); - } - - void IncrementRule(const TRule& rule) { - RuleCRPMap::iterator it = rules.find(rule.f_); - if (it == rules.end()) - it = rules.insert(make_pair(rule.f_, CCRP_NoTable<TRule>(1,1))).first; - if (it->second.increment(rule)) { - base *= (*rp0)(rule); - } - if (src_phrases.increment(rule.f_)) - base *= srcp0(rule.f_); - } - - void IncrementRules(const vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - void IncrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].increment(dist)) - base *= jp0(dist, src_len); - } - - void DecrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].decrement(dist)) - base /= jp0(dist, src_len); - } - - void IncrementJumps(const vector<int>& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - IncrementJump(js[i], src_len); - } - - void DecrementJumps(const vector<int>& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - DecrementJump(js[i], src_len); - } - - // p(jump = dist | src_len , z) - prob_t JumpProbability(int dist, unsigned src_len) { - const prob_t p0 = jp0(dist, src_len); - const double lp = src_jumps[src_len].logprob(dist, log(p0)); - prob_t q; q.logeq(lp); - return q; - } - - // p(rule.f_ | z) * p(rule.e_ | rule.f_ , z) - prob_t RuleProbability(const TRule& rule) const { - const prob_t p0 = (*rp0)(rule); - prob_t srcp; srcp.logeq(src_phrases.logprob(rule.f_, log(srcp0(rule.f_)))); - const RuleCRPMap::const_iterator it = rules.find(rule.f_); - if (it == rules.end()) return srcp * p0; - const double lp = it->second.logprob(rule, log(p0)); - prob_t q; q.logeq(lp); - return q * srcp; - } - - prob_t Likelihood() const { - prob_t p = base; - for (RuleCRPMap::const_iterator it = rules.begin(); - it != rules.end(); ++it) { - prob_t cl; cl.logeq(it->second.log_crp_prob()); - p *= cl; - } - for (unsigned l = 1; l < src_jumps.size(); ++l) { - if (src_jumps[l].num_customers() > 0) { - prob_t q; - q.logeq(src_jumps[l].log_crp_prob()); - p *= q; - } - } - return p; - } - - JumpBase jp0; - const PhraseConditionalBase* rp0; - prob_t base; - typedef unordered_map<vector<WordID>, CCRP_NoTable<TRule>, boost::hash<vector<WordID> > > RuleCRPMap; - RuleCRPMap rules; - CCRP_NoTable<vector<WordID> > src_phrases; - vector<CCRP_NoTable<int> > src_jumps; -}; - -#endif - -struct MyJointModel { - MyJointModel(PhraseJointBase& rcp0) : - rp0(rcp0), base(prob_t::One()), rules(1,1), src_jumps(200, CCRP_NoTable<int>(1,1)) {} - - void DecrementRule(const TRule& rule) { - if (rules.decrement(rule)) - base /= rp0(rule); - } - - void IncrementRule(const TRule& rule) { - if (rules.increment(rule)) - base *= rp0(rule); - } - - void IncrementRules(const vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - void IncrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].increment(dist)) - base *= jp0(dist, src_len); - } - - void DecrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].decrement(dist)) - base /= jp0(dist, src_len); - } - - void IncrementJumps(const vector<int>& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - IncrementJump(js[i], src_len); - } - - void DecrementJumps(const vector<int>& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - DecrementJump(js[i], src_len); - } - - // p(jump = dist | src_len , z) - prob_t JumpProbability(int dist, unsigned src_len) { - const prob_t p0 = jp0(dist, src_len); - const double lp = src_jumps[src_len].logprob(dist, log(p0)); - prob_t q; q.logeq(lp); - return q; - } - - // p(rule.f_ | z) * p(rule.e_ | rule.f_ , z) - prob_t RuleProbability(const TRule& rule) const { - prob_t p; p.logeq(rules.logprob(rule, log(rp0(rule)))); - return p; - } - - prob_t Likelihood() const { - prob_t p = base; - prob_t q; q.logeq(rules.log_crp_prob()); - p *= q; - for (unsigned l = 1; l < src_jumps.size(); ++l) { - if (src_jumps[l].num_customers() > 0) { - prob_t q; - q.logeq(src_jumps[l].log_crp_prob()); - p *= q; - } - } - return p; - } - - JumpBase jp0; - const PhraseJointBase& rp0; - prob_t base; - CCRP_NoTable<TRule> rules; - vector<CCRP_NoTable<int> > src_jumps; -}; - -struct BackwardEstimate { - BackwardEstimate(const Model1& m1, const vector<WordID>& src, const vector<WordID>& trg) : - model1_(m1), src_(src), trg_(trg) { - } - const prob_t& operator()(const vector<bool>& src_cov, unsigned trg_cov) const { - assert(src_.size() == src_cov.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector<WordID> r(src_.size() + 1); r.clear(); - r.push_back(0); // NULL word - for (int i = 0; i < src_cov.size(); ++i) - if (!src_cov[i]) r.push_back(src_[i]); - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(Md::log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - } - return e; - } - const Model1& model1_; - const vector<WordID>& src_; - const vector<WordID>& trg_; - mutable unordered_map<vector<bool>, map<unsigned, prob_t>, boost::hash<vector<bool> > > cache_; -}; - -struct BackwardEstimateSym { - BackwardEstimateSym(const Model1& m1, - const Model1& invm1, const vector<WordID>& src, const vector<WordID>& trg) : - model1_(m1), invmodel1_(invm1), src_(src), trg_(trg) { - } - const prob_t& operator()(const vector<bool>& src_cov, unsigned trg_cov) const { - assert(src_.size() == src_cov.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector<WordID> r(src_.size() + 1); r.clear(); - for (int i = 0; i < src_cov.size(); ++i) - if (!src_cov[i]) r.push_back(src_[i]); - r.push_back(0); // NULL word - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(Md::log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - r.pop_back(); - const prob_t inv_uniform(1.0 / (trg_.size() - trg_cov + 1.0)); - prob_t inv; - inv.logeq(Md::log_poisson(r.size(), trg_.size() - trg_cov)); - for (unsigned i = 0; i < r.size(); ++i) { - prob_t p; - for (unsigned j = trg_cov - 1; j < trg_.size(); ++j) - p += invmodel1_(j < trg_cov ? 0 : trg_[j], r[i]); - if (p.is_0()) { - cerr << "ERROR: p_inv(" << TD::Convert(r[i]) << " | " << TD::GetString(trg_) << ") = 0!\n"; - abort(); - } - p *= inv_uniform; - inv *= p; - } - prob_t x = pow(e * inv, 0.5); - e = x; - //cerr << "Forward: " << log(e) << "\tBackward: " << log(inv) << "\t prop: " << log(x) << endl; - } - return e; - } - const Model1& model1_; - const Model1& invmodel1_; - const vector<WordID>& src_; - const vector<WordID>& trg_; - mutable unordered_map<vector<bool>, map<unsigned, prob_t>, boost::hash<vector<bool> > > cache_; -}; - -struct Particle { - Particle() : weight(prob_t::One()), src_cov(), trg_cov(), prev_pos(-1) {} - prob_t weight; - prob_t gamma_last; - vector<int> src_jumps; - vector<TRulePtr> rules; - vector<bool> src_cv; - int src_cov; - int trg_cov; - int prev_pos; -}; - -ostream& operator<<(ostream& o, const vector<bool>& v) { - for (int i = 0; i < v.size(); ++i) - o << (v[i] ? '1' : '0'); - return o; -} -ostream& operator<<(ostream& o, const Particle& p) { - o << "[cv=" << p.src_cv << " src_cov=" << p.src_cov << " trg_cov=" << p.trg_cov << " last_pos=" << p.prev_pos << " num_rules=" << p.rules.size() << " w=" << log(p.weight) << ']'; - return o; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned kMAX_TRG_PHRASE = conf["max_trg_phrase"].as<unsigned>(); - const unsigned kMAX_SRC_PHRASE = conf["max_src_phrase"].as<unsigned>(); - const unsigned particles = conf["particles"].as<unsigned>(); - const unsigned samples = conf["samples"].as<unsigned>(); - const unsigned rejuv_freq = conf["filter_frequency"].as<unsigned>(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector<vector<WordID> > corpuse, corpusf; - set<WordID> vocabe, vocabf; - cerr << "Reading corpus...\n"; - ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - - const int kLHS = -TD::Convert("X"); - Model1 m1(conf["model1"].as<string>()); - Model1 invm1(conf["inverse_model1"].as<string>()); - -#if 0 - PhraseConditionalBase lp0(m1, conf["model1_interpolation_weight"].as<double>(), vocabe.size()); - MyConditionalModel m(lp0); -#else - PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as<double>(), vocabe.size(), vocabf.size()); - MyJointModel m(lp0); -#endif - - MultinomialResampleFilter<Particle> filter(&rng); - cerr << "Initializing reachability limits...\n"; - vector<Particle> ps(corpusf.size()); - vector<Reachability> reaches; reaches.reserve(corpusf.size()); - for (int ci = 0; ci < corpusf.size(); ++ci) - reaches.push_back(Reachability(corpusf[ci].size(), - corpuse[ci].size(), - kMAX_SRC_PHRASE, - kMAX_TRG_PHRASE)); - cerr << "Sampling...\n"; - vector<Particle> tmp_p(10000); // work space - SampleSet<prob_t> pfss; - for (int SS=0; SS < samples; ++SS) { - for (int ci = 0; ci < corpusf.size(); ++ci) { - vector<int>& src = corpusf[ci]; - vector<int>& trg = corpuse[ci]; - m.DecrementRules(ps[ci].rules); - m.DecrementJumps(ps[ci].src_jumps, src.size()); - - //BackwardEstimate be(m1, src, trg); - BackwardEstimateSym be(m1, invm1, src, trg); - const Reachability& r = reaches[ci]; - vector<Particle> lps(particles); - - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - p.src_cv.resize(src.size(), false); - } - - bool all_complete = false; - while(!all_complete) { - SampleSet<prob_t> ss; - - // all particles have now been extended a bit, we will reweight them now - if (lps[0].trg_cov > 0) - filter(&lps); - - // loop over all particles and extend them - bool done_nothing = true; - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - int tic = 0; - while(p.trg_cov < trg.size() && tic < rejuv_freq) { - ++tic; - done_nothing = false; - ss.clear(); - TRule x; x.lhs_ = kLHS; - prob_t z; - int first_uncovered = src.size(); - int last_uncovered = -1; - for (int i = 0; i < src.size(); ++i) { - const bool is_uncovered = !p.src_cv[i]; - if (i < first_uncovered && is_uncovered) first_uncovered = i; - if (is_uncovered && i > last_uncovered) last_uncovered = i; - } - assert(last_uncovered > -1); - assert(first_uncovered < src.size()); - - for (int trg_len = 1; trg_len <= kMAX_TRG_PHRASE; ++trg_len) { - x.e_.push_back(trg[trg_len - 1 + p.trg_cov]); - for (int src_len = 1; src_len <= kMAX_SRC_PHRASE; ++src_len) { - if (!r.edges[p.src_cov][p.trg_cov][src_len][trg_len]) continue; - - const int last_possible_start = last_uncovered - src_len + 1; - assert(last_possible_start >= 0); - //cerr << src_len << "," << trg_len << " is allowed. E=" << TD::GetString(x.e_) << endl; - //cerr << " first_uncovered=" << first_uncovered << " last_possible_start=" << last_possible_start << endl; - for (int i = first_uncovered; i <= last_possible_start; ++i) { - if (p.src_cv[i]) continue; - assert(ss.size() < tmp_p.size()); // if fails increase tmp_p size - Particle& np = tmp_p[ss.size()]; - np = p; - x.f_.clear(); - int gap_add = 0; - bool bad = false; - prob_t jp = prob_t::One(); - int prev_pos = p.prev_pos; - for (int j = 0; j < src_len; ++j) { - if ((j + i + gap_add) == src.size()) { bad = true; break; } - while ((i+j+gap_add) < src.size() && p.src_cv[i + j + gap_add]) { ++gap_add; } - if ((j + i + gap_add) == src.size()) { bad = true; break; } - np.src_cv[i + j + gap_add] = true; - x.f_.push_back(src[i + j + gap_add]); - jp *= m.JumpProbability(i + j + gap_add - prev_pos, src.size()); - int jump = i + j + gap_add - prev_pos; - assert(jump != 0); - np.src_jumps.push_back(jump); - prev_pos = i + j + gap_add; - } - if (bad) continue; - np.prev_pos = prev_pos; - np.src_cov += x.f_.size(); - np.trg_cov += x.e_.size(); - if (x.f_.size() != src_len) continue; - prob_t rp = m.RuleProbability(x); - np.gamma_last = rp * jp; - const prob_t u = pow(np.gamma_last * be(np.src_cv, np.trg_cov), 0.2); - //cerr << "**rule=" << x << endl; - //cerr << " u=" << log(u) << " rule=" << rp << " jump=" << jp << endl; - ss.add(u); - np.rules.push_back(TRulePtr(new TRule(x))); - z += u; - - const bool completed = (p.trg_cov == trg.size()); - if (completed) { - int last_jump = src.size() - p.prev_pos; - assert(last_jump > 0); - p.src_jumps.push_back(last_jump); - p.weight *= m.JumpProbability(last_jump, src.size()); - } - } - } - } - cerr << "number of edges to consider: " << ss.size() << endl; - const int sampled = rng.SelectSample(ss); - prob_t q_n = ss[sampled] / z; - p = tmp_p[sampled]; - //m.IncrementRule(*p.rules.back()); - p.weight *= p.gamma_last / q_n; - cerr << "[w=" << log(p.weight) << "]\tsampled rule: " << p.rules.back()->AsString() << endl; - cerr << p << endl; - } - } // loop over particles (pi = 0 .. particles) - if (done_nothing) all_complete = true; - } - pfss.clear(); - for (int i = 0; i < lps.size(); ++i) - pfss.add(lps[i].weight); - const int sampled = rng.SelectSample(pfss); - ps[ci] = lps[sampled]; - m.IncrementRules(lps[sampled].rules); - m.IncrementJumps(lps[sampled].src_jumps, src.size()); - for (int i = 0; i < lps[sampled].rules.size(); ++i) { cerr << "S:\t" << lps[sampled].rules[i]->AsString() << "\n"; } - cerr << "tmp-LLH: " << log(m.Likelihood()) << endl; - } - cerr << "LLH: " << log(m.Likelihood()) << endl; - for (int sni = 0; sni < 5; ++sni) { - for (int i = 0; i < ps[sni].rules.size(); ++i) { cerr << "\t" << ps[sni].rules[i]->AsString() << endl; } - } - } - return 0; -} - diff --git a/gi/pf/pfdist.new.cc b/gi/pf/pfdist.new.cc deleted file mode 100644 index 3169eb75..00000000 --- a/gi/pf/pfdist.new.cc +++ /dev/null @@ -1,620 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "base_measures.h" -#include "reachability.h" -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "ccrp_onetable.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -shared_ptr<MT19937> prng; - -size_t hash_value(const TRule& r) { - size_t h = boost::hash_value(r.e_); - boost::hash_combine(h, -r.lhs_); - boost::hash_combine(h, boost::hash_value(r.f_)); - return h; -} - -bool operator==(const TRule& a, const TRule& b) { - return (a.lhs_ == b.lhs_ && a.e_ == b.e_ && a.f_ == b.f_); -} - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("particles,p",po::value<unsigned>()->default_value(25),"Number of particles") - ("input,i",po::value<string>(),"Read parallel data from") - ("max_src_phrase",po::value<unsigned>()->default_value(5),"Maximum length of source language phrases") - ("max_trg_phrase",po::value<unsigned>()->default_value(5),"Maximum length of target language phrases") - ("model1,m",po::value<string>(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value<string>(),"Inverse Model 1 parameters (used in backward estimate)") - ("model1_interpolation_weight",po::value<double>()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -void ReadParallelCorpus(const string& filename, - vector<vector<WordID> >* f, - vector<vector<WordID> >* e, - set<WordID>* vocab_f, - set<WordID>* vocab_e) { - f->clear(); - e->clear(); - vocab_f->clear(); - vocab_e->clear(); - istream* in; - if (filename == "-") - in = &cin; - else - in = new ifstream(filename.c_str()); - assert(*in); - string line; - const WordID kDIV = TD::Convert("|||"); - vector<WordID> tmp; - while(*in) { - getline(*in, line); - if (line.empty() && !*in) break; - e->push_back(vector<int>()); - f->push_back(vector<int>()); - vector<int>& le = e->back(); - vector<int>& lf = f->back(); - tmp.clear(); - TD::ConvertSentence(line, &tmp); - bool isf = true; - for (unsigned i = 0; i < tmp.size(); ++i) { - const int cur = tmp[i]; - if (isf) { - if (kDIV == cur) { isf = false; } else { - lf.push_back(cur); - vocab_f->insert(cur); - } - } else { - assert(cur != kDIV); - le.push_back(cur); - vocab_e->insert(cur); - } - } - assert(isf == false); - } - if (in != &cin) delete in; -} - -#if 0 -struct MyConditionalModel { - MyConditionalModel(PhraseConditionalBase& rcp0) : rp0(&rcp0), base(prob_t::One()), src_phrases(1,1), src_jumps(200, CCRP_NoTable<int>(1,1)) {} - - prob_t srcp0(const vector<WordID>& src) const { - prob_t p(1.0 / 3000.0); - p.poweq(src.size()); - prob_t lenp; lenp.logeq(log_poisson(src.size(), 1.0)); - p *= lenp; - return p; - } - - void DecrementRule(const TRule& rule) { - const RuleCRPMap::iterator it = rules.find(rule.f_); - assert(it != rules.end()); - if (it->second.decrement(rule)) { - base /= (*rp0)(rule); - if (it->second.num_customers() == 0) - rules.erase(it); - } - if (src_phrases.decrement(rule.f_)) - base /= srcp0(rule.f_); - } - - void IncrementRule(const TRule& rule) { - RuleCRPMap::iterator it = rules.find(rule.f_); - if (it == rules.end()) - it = rules.insert(make_pair(rule.f_, CCRP_NoTable<TRule>(1,1))).first; - if (it->second.increment(rule)) { - base *= (*rp0)(rule); - } - if (src_phrases.increment(rule.f_)) - base *= srcp0(rule.f_); - } - - void IncrementRules(const vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - void IncrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].increment(dist)) - base *= jp0(dist, src_len); - } - - void DecrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].decrement(dist)) - base /= jp0(dist, src_len); - } - - void IncrementJumps(const vector<int>& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - IncrementJump(js[i], src_len); - } - - void DecrementJumps(const vector<int>& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - DecrementJump(js[i], src_len); - } - - // p(jump = dist | src_len , z) - prob_t JumpProbability(int dist, unsigned src_len) { - const prob_t p0 = jp0(dist, src_len); - const double lp = src_jumps[src_len].logprob(dist, log(p0)); - prob_t q; q.logeq(lp); - return q; - } - - // p(rule.f_ | z) * p(rule.e_ | rule.f_ , z) - prob_t RuleProbability(const TRule& rule) const { - const prob_t p0 = (*rp0)(rule); - prob_t srcp; srcp.logeq(src_phrases.logprob(rule.f_, log(srcp0(rule.f_)))); - const RuleCRPMap::const_iterator it = rules.find(rule.f_); - if (it == rules.end()) return srcp * p0; - const double lp = it->second.logprob(rule, log(p0)); - prob_t q; q.logeq(lp); - return q * srcp; - } - - prob_t Likelihood() const { - prob_t p = base; - for (RuleCRPMap::const_iterator it = rules.begin(); - it != rules.end(); ++it) { - prob_t cl; cl.logeq(it->second.log_crp_prob()); - p *= cl; - } - for (unsigned l = 1; l < src_jumps.size(); ++l) { - if (src_jumps[l].num_customers() > 0) { - prob_t q; - q.logeq(src_jumps[l].log_crp_prob()); - p *= q; - } - } - return p; - } - - JumpBase jp0; - const PhraseConditionalBase* rp0; - prob_t base; - typedef unordered_map<vector<WordID>, CCRP_NoTable<TRule>, boost::hash<vector<WordID> > > RuleCRPMap; - RuleCRPMap rules; - CCRP_NoTable<vector<WordID> > src_phrases; - vector<CCRP_NoTable<int> > src_jumps; -}; - -#endif - -struct MyJointModel { - MyJointModel(PhraseJointBase& rcp0) : - rp0(rcp0), base(prob_t::One()), rules(1,1), src_jumps(200, CCRP_NoTable<int>(1,1)) {} - - void DecrementRule(const TRule& rule) { - if (rules.decrement(rule)) - base /= rp0(rule); - } - - void IncrementRule(const TRule& rule) { - if (rules.increment(rule)) - base *= rp0(rule); - } - - void IncrementRules(const vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - void IncrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].increment(dist)) - base *= jp0(dist, src_len); - } - - void DecrementJump(int dist, unsigned src_len) { - assert(src_len > 0); - if (src_jumps[src_len].decrement(dist)) - base /= jp0(dist, src_len); - } - - void IncrementJumps(const vector<int>& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - IncrementJump(js[i], src_len); - } - - void DecrementJumps(const vector<int>& js, unsigned src_len) { - for (unsigned i = 0; i < js.size(); ++i) - DecrementJump(js[i], src_len); - } - - // p(jump = dist | src_len , z) - prob_t JumpProbability(int dist, unsigned src_len) { - const prob_t p0 = jp0(dist, src_len); - const double lp = src_jumps[src_len].logprob(dist, log(p0)); - prob_t q; q.logeq(lp); - return q; - } - - // p(rule.f_ | z) * p(rule.e_ | rule.f_ , z) - prob_t RuleProbability(const TRule& rule) const { - prob_t p; p.logeq(rules.logprob(rule, log(rp0(rule)))); - return p; - } - - prob_t Likelihood() const { - prob_t p = base; - prob_t q; q.logeq(rules.log_crp_prob()); - p *= q; - for (unsigned l = 1; l < src_jumps.size(); ++l) { - if (src_jumps[l].num_customers() > 0) { - prob_t q; - q.logeq(src_jumps[l].log_crp_prob()); - p *= q; - } - } - return p; - } - - JumpBase jp0; - const PhraseJointBase& rp0; - prob_t base; - CCRP_NoTable<TRule> rules; - vector<CCRP_NoTable<int> > src_jumps; -}; - -struct BackwardEstimate { - BackwardEstimate(const Model1& m1, const vector<WordID>& src, const vector<WordID>& trg) : - model1_(m1), src_(src), trg_(trg) { - } - const prob_t& operator()(const vector<bool>& src_cov, unsigned trg_cov) const { - assert(src_.size() == src_cov.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector<WordID> r(src_.size() + 1); r.clear(); - r.push_back(0); // NULL word - for (int i = 0; i < src_cov.size(); ++i) - if (!src_cov[i]) r.push_back(src_[i]); - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - } - return e; - } - const Model1& model1_; - const vector<WordID>& src_; - const vector<WordID>& trg_; - mutable unordered_map<vector<bool>, map<unsigned, prob_t>, boost::hash<vector<bool> > > cache_; -}; - -struct BackwardEstimateSym { - BackwardEstimateSym(const Model1& m1, - const Model1& invm1, const vector<WordID>& src, const vector<WordID>& trg) : - model1_(m1), invmodel1_(invm1), src_(src), trg_(trg) { - } - const prob_t& operator()(const vector<bool>& src_cov, unsigned trg_cov) const { - assert(src_.size() == src_cov.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector<WordID> r(src_.size() + 1); r.clear(); - for (int i = 0; i < src_cov.size(); ++i) - if (!src_cov[i]) r.push_back(src_[i]); - r.push_back(0); // NULL word - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - r.pop_back(); - const prob_t inv_uniform(1.0 / (trg_.size() - trg_cov + 1.0)); - prob_t inv; - inv.logeq(log_poisson(r.size(), trg_.size() - trg_cov)); - for (unsigned i = 0; i < r.size(); ++i) { - prob_t p; - for (unsigned j = trg_cov - 1; j < trg_.size(); ++j) - p += invmodel1_(j < trg_cov ? 0 : trg_[j], r[i]); - if (p.is_0()) { - cerr << "ERROR: p_inv(" << TD::Convert(r[i]) << " | " << TD::GetString(trg_) << ") = 0!\n"; - abort(); - } - p *= inv_uniform; - inv *= p; - } - prob_t x = pow(e * inv, 0.5); - e = x; - //cerr << "Forward: " << log(e) << "\tBackward: " << log(inv) << "\t prop: " << log(x) << endl; - } - return e; - } - const Model1& model1_; - const Model1& invmodel1_; - const vector<WordID>& src_; - const vector<WordID>& trg_; - mutable unordered_map<vector<bool>, map<unsigned, prob_t>, boost::hash<vector<bool> > > cache_; -}; - -struct Particle { - Particle() : weight(prob_t::One()), src_cov(), trg_cov(), prev_pos(-1) {} - prob_t weight; - prob_t gamma_last; - vector<int> src_jumps; - vector<TRulePtr> rules; - vector<bool> src_cv; - int src_cov; - int trg_cov; - int prev_pos; -}; - -ostream& operator<<(ostream& o, const vector<bool>& v) { - for (int i = 0; i < v.size(); ++i) - o << (v[i] ? '1' : '0'); - return o; -} -ostream& operator<<(ostream& o, const Particle& p) { - o << "[cv=" << p.src_cv << " src_cov=" << p.src_cov << " trg_cov=" << p.trg_cov << " last_pos=" << p.prev_pos << " num_rules=" << p.rules.size() << " w=" << log(p.weight) << ']'; - return o; -} - -void FilterCrapParticlesAndReweight(vector<Particle>* pps) { - vector<Particle>& ps = *pps; - SampleSet<prob_t> ss; - for (int i = 0; i < ps.size(); ++i) - ss.add(ps[i].weight); - vector<Particle> nps; nps.reserve(ps.size()); - const prob_t uniform_weight(1.0 / ps.size()); - for (int i = 0; i < ps.size(); ++i) { - nps.push_back(ps[prng->SelectSample(ss)]); - nps[i].weight = uniform_weight; - } - nps.swap(ps); -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned kMAX_TRG_PHRASE = conf["max_trg_phrase"].as<unsigned>(); - const unsigned kMAX_SRC_PHRASE = conf["max_src_phrase"].as<unsigned>(); - const unsigned particles = conf["particles"].as<unsigned>(); - const unsigned samples = conf["samples"].as<unsigned>(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector<vector<WordID> > corpuse, corpusf; - set<WordID> vocabe, vocabf; - cerr << "Reading corpus...\n"; - ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - - const int kLHS = -TD::Convert("X"); - Model1 m1(conf["model1"].as<string>()); - Model1 invm1(conf["inverse_model1"].as<string>()); - -#if 0 - PhraseConditionalBase lp0(m1, conf["model1_interpolation_weight"].as<double>(), vocabe.size()); - MyConditionalModel m(lp0); -#else - PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as<double>(), vocabe.size(), vocabf.size()); - MyJointModel m(lp0); -#endif - - cerr << "Initializing reachability limits...\n"; - vector<Particle> ps(corpusf.size()); - vector<Reachability> reaches; reaches.reserve(corpusf.size()); - for (int ci = 0; ci < corpusf.size(); ++ci) - reaches.push_back(Reachability(corpusf[ci].size(), - corpuse[ci].size(), - kMAX_SRC_PHRASE, - kMAX_TRG_PHRASE)); - cerr << "Sampling...\n"; - vector<Particle> tmp_p(10000); // work space - SampleSet<prob_t> pfss; - for (int SS=0; SS < samples; ++SS) { - for (int ci = 0; ci < corpusf.size(); ++ci) { - vector<int>& src = corpusf[ci]; - vector<int>& trg = corpuse[ci]; - m.DecrementRules(ps[ci].rules); - m.DecrementJumps(ps[ci].src_jumps, src.size()); - - //BackwardEstimate be(m1, src, trg); - BackwardEstimateSym be(m1, invm1, src, trg); - const Reachability& r = reaches[ci]; - vector<Particle> lps(particles); - - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - p.src_cv.resize(src.size(), false); - } - - bool all_complete = false; - while(!all_complete) { - SampleSet<prob_t> ss; - - // all particles have now been extended a bit, we will reweight them now - if (lps[0].trg_cov > 0) - FilterCrapParticlesAndReweight(&lps); - - // loop over all particles and extend them - bool done_nothing = true; - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - int tic = 0; - const int rejuv_freq = 1; - while(p.trg_cov < trg.size() && tic < rejuv_freq) { - ++tic; - done_nothing = false; - ss.clear(); - TRule x; x.lhs_ = kLHS; - prob_t z; - int first_uncovered = src.size(); - int last_uncovered = -1; - for (int i = 0; i < src.size(); ++i) { - const bool is_uncovered = !p.src_cv[i]; - if (i < first_uncovered && is_uncovered) first_uncovered = i; - if (is_uncovered && i > last_uncovered) last_uncovered = i; - } - assert(last_uncovered > -1); - assert(first_uncovered < src.size()); - - for (int trg_len = 1; trg_len <= kMAX_TRG_PHRASE; ++trg_len) { - x.e_.push_back(trg[trg_len - 1 + p.trg_cov]); - for (int src_len = 1; src_len <= kMAX_SRC_PHRASE; ++src_len) { - if (!r.edges[p.src_cov][p.trg_cov][src_len][trg_len]) continue; - - const int last_possible_start = last_uncovered - src_len + 1; - assert(last_possible_start >= 0); - //cerr << src_len << "," << trg_len << " is allowed. E=" << TD::GetString(x.e_) << endl; - //cerr << " first_uncovered=" << first_uncovered << " last_possible_start=" << last_possible_start << endl; - for (int i = first_uncovered; i <= last_possible_start; ++i) { - if (p.src_cv[i]) continue; - assert(ss.size() < tmp_p.size()); // if fails increase tmp_p size - Particle& np = tmp_p[ss.size()]; - np = p; - x.f_.clear(); - int gap_add = 0; - bool bad = false; - prob_t jp = prob_t::One(); - int prev_pos = p.prev_pos; - for (int j = 0; j < src_len; ++j) { - if ((j + i + gap_add) == src.size()) { bad = true; break; } - while ((i+j+gap_add) < src.size() && p.src_cv[i + j + gap_add]) { ++gap_add; } - if ((j + i + gap_add) == src.size()) { bad = true; break; } - np.src_cv[i + j + gap_add] = true; - x.f_.push_back(src[i + j + gap_add]); - jp *= m.JumpProbability(i + j + gap_add - prev_pos, src.size()); - int jump = i + j + gap_add - prev_pos; - assert(jump != 0); - np.src_jumps.push_back(jump); - prev_pos = i + j + gap_add; - } - if (bad) continue; - np.prev_pos = prev_pos; - np.src_cov += x.f_.size(); - np.trg_cov += x.e_.size(); - if (x.f_.size() != src_len) continue; - prob_t rp = m.RuleProbability(x); - np.gamma_last = rp * jp; - const prob_t u = pow(np.gamma_last * be(np.src_cv, np.trg_cov), 0.2); - //cerr << "**rule=" << x << endl; - //cerr << " u=" << log(u) << " rule=" << rp << " jump=" << jp << endl; - ss.add(u); - np.rules.push_back(TRulePtr(new TRule(x))); - z += u; - - const bool completed = (p.trg_cov == trg.size()); - if (completed) { - int last_jump = src.size() - p.prev_pos; - assert(last_jump > 0); - p.src_jumps.push_back(last_jump); - p.weight *= m.JumpProbability(last_jump, src.size()); - } - } - } - } - cerr << "number of edges to consider: " << ss.size() << endl; - const int sampled = rng.SelectSample(ss); - prob_t q_n = ss[sampled] / z; - p = tmp_p[sampled]; - //m.IncrementRule(*p.rules.back()); - p.weight *= p.gamma_last / q_n; - cerr << "[w=" << log(p.weight) << "]\tsampled rule: " << p.rules.back()->AsString() << endl; - cerr << p << endl; - } - } // loop over particles (pi = 0 .. particles) - if (done_nothing) all_complete = true; - } - pfss.clear(); - for (int i = 0; i < lps.size(); ++i) - pfss.add(lps[i].weight); - const int sampled = rng.SelectSample(pfss); - ps[ci] = lps[sampled]; - m.IncrementRules(lps[sampled].rules); - m.IncrementJumps(lps[sampled].src_jumps, src.size()); - for (int i = 0; i < lps[sampled].rules.size(); ++i) { cerr << "S:\t" << lps[sampled].rules[i]->AsString() << "\n"; } - cerr << "tmp-LLH: " << log(m.Likelihood()) << endl; - } - cerr << "LLH: " << log(m.Likelihood()) << endl; - for (int sni = 0; sni < 5; ++sni) { - for (int i = 0; i < ps[sni].rules.size(); ++i) { cerr << "\t" << ps[sni].rules[i]->AsString() << endl; } - } - } - return 0; -} - diff --git a/gi/pf/pfnaive.cc b/gi/pf/pfnaive.cc deleted file mode 100644 index 958ec4e2..00000000 --- a/gi/pf/pfnaive.cc +++ /dev/null @@ -1,284 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "pf.h" -#include "base_distributions.h" -#include "monotonic_pseg.h" -#include "reachability.h" -#include "viterbi.h" -#include "hg.h" -#include "trule.h" -#include "tdict.h" -#include "filelib.h" -#include "dict.h" -#include "sampler.h" -#include "ccrp_nt.h" -#include "ccrp_onetable.h" -#include "corpus.h" - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr<MT19937> prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,s",po::value<unsigned>()->default_value(1000),"Number of samples") - ("particles,p",po::value<unsigned>()->default_value(30),"Number of particles") - ("filter_frequency,f",po::value<unsigned>()->default_value(5),"Number of time steps between filterings") - ("input,i",po::value<string>(),"Read parallel data from") - ("max_src_phrase",po::value<unsigned>()->default_value(5),"Maximum length of source language phrases") - ("max_trg_phrase",po::value<unsigned>()->default_value(5),"Maximum length of target language phrases") - ("model1,m",po::value<string>(),"Model 1 parameters (used in base distribution)") - ("inverse_model1,M",po::value<string>(),"Inverse Model 1 parameters (used in backward estimate)") - ("model1_interpolation_weight",po::value<double>()->default_value(0.95),"Mixing proportion of model 1 with uniform target distribution") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help,h", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("input") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -struct BackwardEstimateSym { - BackwardEstimateSym(const Model1& m1, - const Model1& invm1, const vector<WordID>& src, const vector<WordID>& trg) : - model1_(m1), invmodel1_(invm1), src_(src), trg_(trg) { - } - const prob_t& operator()(unsigned src_cov, unsigned trg_cov) const { - assert(src_cov <= src_.size()); - assert(trg_cov <= trg_.size()); - prob_t& e = cache_[src_cov][trg_cov]; - if (e.is_0()) { - if (trg_cov == trg_.size()) { e = prob_t::One(); return e; } - vector<WordID> r(src_.size() + 1); r.clear(); - for (int i = src_cov; i < src_.size(); ++i) - r.push_back(src_[i]); - r.push_back(0); // NULL word - const prob_t uniform_alignment(1.0 / r.size()); - e.logeq(Md::log_poisson(trg_.size() - trg_cov, r.size() - 1)); // p(trg len remaining | src len remaining) - for (unsigned j = trg_cov; j < trg_.size(); ++j) { - prob_t p; - for (unsigned i = 0; i < r.size(); ++i) - p += model1_(r[i], trg_[j]); - if (p.is_0()) { - cerr << "ERROR: p(" << TD::Convert(trg_[j]) << " | " << TD::GetString(r) << ") = 0!\n"; - abort(); - } - p *= uniform_alignment; - e *= p; - } - r.pop_back(); - const prob_t inv_uniform(1.0 / (trg_.size() - trg_cov + 1.0)); - prob_t inv; - inv.logeq(Md::log_poisson(r.size(), trg_.size() - trg_cov)); - for (unsigned i = 0; i < r.size(); ++i) { - prob_t p; - for (unsigned j = trg_cov - 1; j < trg_.size(); ++j) - p += invmodel1_(j < trg_cov ? 0 : trg_[j], r[i]); - if (p.is_0()) { - cerr << "ERROR: p_inv(" << TD::Convert(r[i]) << " | " << TD::GetString(trg_) << ") = 0!\n"; - abort(); - } - p *= inv_uniform; - inv *= p; - } - prob_t x = pow(e * inv, 0.5); - e = x; - //cerr << "Forward: " << log(e) << "\tBackward: " << log(inv) << "\t prop: " << log(x) << endl; - } - return e; - } - const Model1& model1_; - const Model1& invmodel1_; - const vector<WordID>& src_; - const vector<WordID>& trg_; - mutable unordered_map<unsigned, map<unsigned, prob_t> > cache_; -}; - -struct Particle { - Particle() : weight(prob_t::One()), src_cov(), trg_cov() {} - prob_t weight; - prob_t gamma_last; - vector<TRulePtr> rules; - int src_cov; - int trg_cov; -}; - -ostream& operator<<(ostream& o, const vector<bool>& v) { - for (int i = 0; i < v.size(); ++i) - o << (v[i] ? '1' : '0'); - return o; -} -ostream& operator<<(ostream& o, const Particle& p) { - o << "[src_cov=" << p.src_cov << " trg_cov=" << p.trg_cov << " num_rules=" << p.rules.size() << " w=" << log(p.weight) << ']'; - return o; -} - -int main(int argc, char** argv) { - po::variables_map conf; - InitCommandLine(argc, argv, &conf); - const unsigned kMAX_TRG_PHRASE = conf["max_trg_phrase"].as<unsigned>(); - const unsigned kMAX_SRC_PHRASE = conf["max_src_phrase"].as<unsigned>(); - const unsigned particles = conf["particles"].as<unsigned>(); - const unsigned samples = conf["samples"].as<unsigned>(); - const unsigned rejuv_freq = conf["filter_frequency"].as<unsigned>(); - - if (!conf.count("model1")) { - cerr << argv[0] << "Please use --model1 to specify model 1 parameters\n"; - return 1; - } - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - - vector<vector<WordID> > corpuse, corpusf; - set<WordID> vocabe, vocabf; - cerr << "Reading corpus...\n"; - corpus::ReadParallelCorpus(conf["input"].as<string>(), &corpusf, &corpuse, &vocabf, &vocabe); - cerr << "F-corpus size: " << corpusf.size() << " sentences\t (" << vocabf.size() << " word types)\n"; - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - assert(corpusf.size() == corpuse.size()); - - const int kLHS = -TD::Convert("X"); - Model1 m1(conf["model1"].as<string>()); - Model1 invm1(conf["inverse_model1"].as<string>()); - - PhraseJointBase lp0(m1, conf["model1_interpolation_weight"].as<double>(), vocabe.size(), vocabf.size()); - PhraseJointBase_BiDir alp0(m1, invm1, conf["model1_interpolation_weight"].as<double>(), vocabe.size(), vocabf.size()); - MonotonicParallelSegementationModel<PhraseJointBase_BiDir> m(alp0); - TRule xx("[X] ||| ms. kimura ||| MS. KIMURA ||| X=0"); - cerr << xx << endl << lp0(xx) << " " << alp0(xx) << endl; - TRule xx12("[X] ||| . ||| PHARMACY . ||| X=0"); - TRule xx21("[X] ||| pharmacy . ||| . ||| X=0"); -// TRule xx22("[X] ||| . ||| . ||| X=0"); - TRule xx22("[X] ||| . ||| THE . ||| X=0"); - cerr << xx12 << "\t" << lp0(xx12) << " " << alp0(xx12) << endl; - cerr << xx21 << "\t" << lp0(xx21) << " " << alp0(xx21) << endl; - cerr << xx22 << "\t" << lp0(xx22) << " " << alp0(xx22) << endl; - - cerr << "Initializing reachability limits...\n"; - vector<Particle> ps(corpusf.size()); - vector<Reachability> reaches; reaches.reserve(corpusf.size()); - for (int ci = 0; ci < corpusf.size(); ++ci) - reaches.push_back(Reachability(corpusf[ci].size(), - corpuse[ci].size(), - kMAX_SRC_PHRASE, - kMAX_TRG_PHRASE)); - cerr << "Sampling...\n"; - vector<Particle> tmp_p(10000); // work space - SampleSet<prob_t> pfss; - SystematicResampleFilter<Particle> filter(&rng); - // MultinomialResampleFilter<Particle> filter(&rng); - for (int SS=0; SS < samples; ++SS) { - for (int ci = 0; ci < corpusf.size(); ++ci) { - vector<int>& src = corpusf[ci]; - vector<int>& trg = corpuse[ci]; - m.DecrementRulesAndStops(ps[ci].rules); - const prob_t q_stop = m.StopProbability(); - const prob_t q_cont = m.ContinueProbability(); - cerr << "P(stop)=" << q_stop << "\tP(continue)=" <<q_cont << endl; - - BackwardEstimateSym be(m1, invm1, src, trg); - const Reachability& r = reaches[ci]; - vector<Particle> lps(particles); - - bool all_complete = false; - while(!all_complete) { - SampleSet<prob_t> ss; - - // all particles have now been extended a bit, we will reweight them now - if (lps[0].trg_cov > 0) - filter(&lps); - - // loop over all particles and extend them - bool done_nothing = true; - for (int pi = 0; pi < particles; ++pi) { - Particle& p = lps[pi]; - int tic = 0; - while(p.trg_cov < trg.size() && tic < rejuv_freq) { - ++tic; - done_nothing = false; - ss.clear(); - TRule x; x.lhs_ = kLHS; - prob_t z; - - for (int trg_len = 1; trg_len <= kMAX_TRG_PHRASE; ++trg_len) { - x.e_.push_back(trg[trg_len - 1 + p.trg_cov]); - for (int src_len = 1; src_len <= kMAX_SRC_PHRASE; ++src_len) { - if (!r.edges[p.src_cov][p.trg_cov][src_len][trg_len]) continue; - - int i = p.src_cov; - assert(ss.size() < tmp_p.size()); // if fails increase tmp_p size - Particle& np = tmp_p[ss.size()]; - np = p; - x.f_.clear(); - for (int j = 0; j < src_len; ++j) - x.f_.push_back(src[i + j]); - np.src_cov += x.f_.size(); - np.trg_cov += x.e_.size(); - const bool stop_now = (np.src_cov == src_len && np.trg_cov == trg_len); - prob_t rp = m.RuleProbability(x) * (stop_now ? q_stop : q_cont); - np.gamma_last = rp; - const prob_t u = pow(np.gamma_last * pow(be(np.src_cov, np.trg_cov), 1.2), 0.1); - //cerr << "**rule=" << x << endl; - //cerr << " u=" << log(u) << " rule=" << rp << endl; - ss.add(u); - np.rules.push_back(TRulePtr(new TRule(x))); - z += u; - } - } - //cerr << "number of edges to consider: " << ss.size() << endl; - const int sampled = rng.SelectSample(ss); - prob_t q_n = ss[sampled] / z; - p = tmp_p[sampled]; - //m.IncrementRule(*p.rules.back()); - p.weight *= p.gamma_last / q_n; - //cerr << "[w=" << log(p.weight) << "]\tsampled rule: " << p.rules.back()->AsString() << endl; - //cerr << p << endl; - } - } // loop over particles (pi = 0 .. particles) - if (done_nothing) all_complete = true; - prob_t wv = prob_t::Zero(); - for (int pp = 0; pp < lps.size(); ++pp) - wv += lps[pp].weight; - for (int pp = 0; pp < lps.size(); ++pp) - lps[pp].weight /= wv; - } - pfss.clear(); - for (int i = 0; i < lps.size(); ++i) - pfss.add(lps[i].weight); - const int sampled = rng.SelectSample(pfss); - ps[ci] = lps[sampled]; - m.IncrementRulesAndStops(lps[sampled].rules); - for (int i = 0; i < lps[sampled].rules.size(); ++i) { cerr << "S:\t" << lps[sampled].rules[i]->AsString() << "\n"; } - cerr << "tmp-LLH: " << log(m.Likelihood()) << endl; - } - cerr << "LLH: " << log(m.Likelihood()) << endl; - } - return 0; -} - diff --git a/gi/pf/poisson_uniform_word_model.h b/gi/pf/poisson_uniform_word_model.h deleted file mode 100644 index 76204a0e..00000000 --- a/gi/pf/poisson_uniform_word_model.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef _POISSON_UNIFORM_WORD_MODEL_H_ -#define _POISSON_UNIFORM_WORD_MODEL_H_ - -#include <cmath> -#include <vector> -#include "prob.h" -#include "m.h" - -// len ~ Poisson(lambda) -// for (1..len) -// e_i ~ Uniform({Vocabulary}) -struct PoissonUniformWordModel { - explicit PoissonUniformWordModel(const unsigned vocab_size, - const unsigned alphabet_size, - const double mean_len = 5) : - lh(prob_t::One()), - v0(-std::log(vocab_size)), - u0(-std::log(alphabet_size)), - mean_length(mean_len) {} - - void ResampleHyperparameters(MT19937*) {} - - inline prob_t operator()(const std::vector<WordID>& s) const { - prob_t p; - p.logeq(Md::log_poisson(s.size(), mean_length) + s.size() * u0); - //p.logeq(v0); - return p; - } - - inline void Increment(const std::vector<WordID>& w, MT19937*) { - lh *= (*this)(w); - } - - inline void Decrement(const std::vector<WordID>& w, MT19937 *) { - lh /= (*this)(w); - } - - inline prob_t Likelihood() const { return lh; } - - void Summary() const {} - - private: - - prob_t lh; // keeps track of the draws from the base distribution - const double v0; // uniform log prob of generating a word - const double u0; // uniform log prob of generating a letter - const double mean_length; // mean length of a word in the base distribution -}; - -#endif diff --git a/gi/pf/pyp_lm.cc b/gi/pf/pyp_lm.cc deleted file mode 100644 index 605d8206..00000000 --- a/gi/pf/pyp_lm.cc +++ /dev/null @@ -1,273 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include <boost/functional.hpp> -#include <boost/program_options.hpp> -#include <boost/program_options/variables_map.hpp> - -#include "gamma_poisson.h" -#include "corpus_tools.h" -#include "m.h" -#include "tdict.h" -#include "sampler.h" -#include "ccrp.h" -#include "tied_resampler.h" - -// A not very memory-efficient implementation of an N-gram LM based on PYPs -// as described in Y.-W. Teh. (2006) A Hierarchical Bayesian Language Model -// based on Pitman-Yor Processes. In Proc. ACL. - -// I use templates to handle the recursive formalation of the prior, so -// the order of the model has to be specified here, at compile time: -#define kORDER 3 - -using namespace std; -using namespace tr1; -namespace po = boost::program_options; - -boost::shared_ptr<MT19937> prng; - -void InitCommandLine(int argc, char** argv, po::variables_map* conf) { - po::options_description opts("Configuration options"); - opts.add_options() - ("samples,n",po::value<unsigned>()->default_value(300),"Number of samples") - ("train,i",po::value<string>(),"Training data file") - ("test,T",po::value<string>(),"Test data file") - ("discount_prior_a,a",po::value<double>()->default_value(1.0), "discount ~ Beta(a,b): a=this") - ("discount_prior_b,b",po::value<double>()->default_value(1.0), "discount ~ Beta(a,b): b=this") - ("strength_prior_s,s",po::value<double>()->default_value(1.0), "strength ~ Gamma(s,r): s=this") - ("strength_prior_r,r",po::value<double>()->default_value(1.0), "strength ~ Gamma(s,r): r=this") - ("random_seed,S",po::value<uint32_t>(), "Random seed"); - po::options_description clo("Command line options"); - clo.add_options() - ("config", po::value<string>(), "Configuration file") - ("help", "Print this help message and exit"); - po::options_description dconfig_options, dcmdline_options; - dconfig_options.add(opts); - dcmdline_options.add(opts).add(clo); - - po::store(parse_command_line(argc, argv, dcmdline_options), *conf); - if (conf->count("config")) { - ifstream config((*conf)["config"].as<string>().c_str()); - po::store(po::parse_config_file(config, dconfig_options), *conf); - } - po::notify(*conf); - - if (conf->count("help") || (conf->count("train") == 0)) { - cerr << dcmdline_options << endl; - exit(1); - } -} - -// uniform distribution over a fixed vocabulary -struct UniformVocabulary { - UniformVocabulary(unsigned vs, double, double, double, double) : p0(1.0 / vs), draws() {} - void increment(WordID, const vector<WordID>&, MT19937*) { ++draws; } - void decrement(WordID, const vector<WordID>&, MT19937*) { --draws; assert(draws >= 0); } - double prob(WordID, const vector<WordID>&) const { return p0; } - void resample_hyperparameters(MT19937*) {} - double log_likelihood() const { return draws * log(p0); } - const double p0; - int draws; -}; - -// Lord Rothschild. 1986. THE DISTRIBUTION OF ENGLISH DICTIONARY WORD LENGTHS. -// Journal of Statistical Planning and Inference 14 (1986) 311-322 -struct PoissonLengthUniformCharWordModel { - explicit PoissonLengthUniformCharWordModel(unsigned vocab_size, double, double, double, double) : plen(5,5), uc(-log(95)), llh() {} - void increment(WordID w, const vector<WordID>& v, MT19937*) { - llh += log(prob(w, v)); // this isn't quite right - plen.increment(TD::Convert(w).size() - 1); - } - void decrement(WordID w, const vector<WordID>& v, MT19937*) { - plen.decrement(TD::Convert(w).size() - 1); - llh -= log(prob(w, v)); // this isn't quite right - } - double prob(WordID w, const vector<WordID>&) const { - const unsigned len = TD::Convert(w).size(); - return plen.prob(len - 1) * exp(uc * len); - } - double log_likelihood() const { return llh; } - void resample_hyperparameters(MT19937*) {} - GammaPoisson plen; - const double uc; - double llh; -}; - -struct PYPAdaptedPoissonLengthUniformCharWordModel { - explicit PYPAdaptedPoissonLengthUniformCharWordModel(unsigned vocab_size, double, double, double, double) : - base(vocab_size,1,1,1,1), - crp(1,1,1,1) {} - void increment(WordID w, const vector<WordID>& v, MT19937* rng) { - double p0 = base.prob(w, v); - if (crp.increment(w, p0, rng)) - base.increment(w, v, rng); - } - void decrement(WordID w, const vector<WordID>& v, MT19937* rng) { - if (crp.decrement(w, rng)) - base.decrement(w, v, rng); - } - double prob(WordID w, const vector<WordID>& v) const { - double p0 = base.prob(w, v); - return crp.prob(w, p0); - } - double log_likelihood() const { return crp.log_crp_prob() + base.log_likelihood(); } - void resample_hyperparameters(MT19937* rng) { crp.resample_hyperparameters(rng); } - PoissonLengthUniformCharWordModel base; - CCRP<WordID> crp; -}; - -template <unsigned N> struct PYPLM; - -#if 1 -template<> struct PYPLM<0> : public UniformVocabulary { - PYPLM(unsigned vs, double a, double b, double c, double d) : - UniformVocabulary(vs, a, b, c, d) {} -}; -#else -#if 0 -template<> struct PYPLM<0> : public PoissonLengthUniformCharWordModel { - PYPLM(unsigned vs, double a, double b, double c, double d) : - PoissonLengthUniformCharWordModel(vs, a, b, c, d) {} -}; -#else -template<> struct PYPLM<0> : public PYPAdaptedPoissonLengthUniformCharWordModel { - PYPLM(unsigned vs, double a, double b, double c, double d) : - PYPAdaptedPoissonLengthUniformCharWordModel(vs, a, b, c, d) {} -}; -#endif -#endif - -// represents an N-gram LM -template <unsigned N> struct PYPLM { - PYPLM(unsigned vs, double da, double db, double ss, double sr) : - backoff(vs, da, db, ss, sr), - tr(da, db, ss, sr, 0.8, 1.0), - lookup(N-1) {} - void increment(WordID w, const vector<WordID>& context, MT19937* rng) { - const double bo = backoff.prob(w, context); - for (unsigned i = 0; i < N-1; ++i) - lookup[i] = context[context.size() - 1 - i]; - typename unordered_map<vector<WordID>, CCRP<WordID>, boost::hash<vector<WordID> > >::iterator it = p.find(lookup); - if (it == p.end()) { - it = p.insert(make_pair(lookup, CCRP<WordID>(0.5,1))).first; - tr.Add(&it->second); // add to resampler - } - if (it->second.increment(w, bo, rng)) - backoff.increment(w, context, rng); - } - void decrement(WordID w, const vector<WordID>& context, MT19937* rng) { - for (unsigned i = 0; i < N-1; ++i) - lookup[i] = context[context.size() - 1 - i]; - typename unordered_map<vector<WordID>, CCRP<WordID>, boost::hash<vector<WordID> > >::iterator it = p.find(lookup); - assert(it != p.end()); - if (it->second.decrement(w, rng)) - backoff.decrement(w, context, rng); - } - double prob(WordID w, const vector<WordID>& context) const { - const double bo = backoff.prob(w, context); - for (unsigned i = 0; i < N-1; ++i) - lookup[i] = context[context.size() - 1 - i]; - typename unordered_map<vector<WordID>, CCRP<WordID>, boost::hash<vector<WordID> > >::const_iterator it = p.find(lookup); - if (it == p.end()) return bo; - return it->second.prob(w, bo); - } - - double log_likelihood() const { - double llh = backoff.log_likelihood(); - typename unordered_map<vector<WordID>, CCRP<WordID>, boost::hash<vector<WordID> > >::const_iterator it; - for (it = p.begin(); it != p.end(); ++it) - llh += it->second.log_crp_prob(); - llh += tr.LogLikelihood(); - return llh; - } - - void resample_hyperparameters(MT19937* rng) { - tr.ResampleHyperparameters(rng); - backoff.resample_hyperparameters(rng); - } - - PYPLM<N-1> backoff; - TiedResampler<CCRP<WordID> > tr; - double discount_a, discount_b, strength_s, strength_r; - double d, strength; - mutable vector<WordID> lookup; // thread-local - unordered_map<vector<WordID>, CCRP<WordID>, boost::hash<vector<WordID> > > p; -}; - -int main(int argc, char** argv) { - po::variables_map conf; - - InitCommandLine(argc, argv, &conf); - const unsigned samples = conf["samples"].as<unsigned>(); - if (conf.count("random_seed")) - prng.reset(new MT19937(conf["random_seed"].as<uint32_t>())); - else - prng.reset(new MT19937); - MT19937& rng = *prng; - vector<vector<WordID> > corpuse; - set<WordID> vocabe; - const WordID kEOS = TD::Convert("</s>"); - cerr << "Reading corpus...\n"; - CorpusTools::ReadFromFile(conf["train"].as<string>(), &corpuse, &vocabe); - cerr << "E-corpus size: " << corpuse.size() << " sentences\t (" << vocabe.size() << " word types)\n"; - vector<vector<WordID> > test; - if (conf.count("test")) - CorpusTools::ReadFromFile(conf["test"].as<string>(), &test); - else - test = corpuse; - PYPLM<kORDER> lm(vocabe.size(), - conf["discount_prior_a"].as<double>(), - conf["discount_prior_b"].as<double>(), - conf["strength_prior_s"].as<double>(), - conf["strength_prior_r"].as<double>()); - vector<WordID> ctx(kORDER - 1, TD::Convert("<s>")); - for (int SS=0; SS < samples; ++SS) { - for (int ci = 0; ci < corpuse.size(); ++ci) { - ctx.resize(kORDER - 1); - const vector<WordID>& s = corpuse[ci]; - for (int i = 0; i <= s.size(); ++i) { - WordID w = (i < s.size() ? s[i] : kEOS); - if (SS > 0) lm.decrement(w, ctx, &rng); - lm.increment(w, ctx, &rng); - ctx.push_back(w); - } - } - if (SS % 10 == 9) { - cerr << " [LLH=" << lm.log_likelihood() << "]" << endl; - if (SS % 30 == 29) lm.resample_hyperparameters(&rng); - } else { cerr << '.' << flush; } - } - double llh = 0; - unsigned cnt = 0; - unsigned oovs = 0; - for (int ci = 0; ci < test.size(); ++ci) { - ctx.resize(kORDER - 1); - const vector<WordID>& s = test[ci]; - for (int i = 0; i <= s.size(); ++i) { - WordID w = (i < s.size() ? s[i] : kEOS); - double lp = log(lm.prob(w, ctx)) / log(2); - if (i < s.size() && vocabe.count(w) == 0) { - cerr << "**OOV "; - ++oovs; - lp = 0; - } - cerr << "p(" << TD::Convert(w) << " |"; - for (int j = ctx.size() + 1 - kORDER; j < ctx.size(); ++j) - cerr << ' ' << TD::Convert(ctx[j]); - cerr << ") = " << lp << endl; - ctx.push_back(w); - llh -= lp; - cnt++; - } - } - cerr << " Log_10 prob: " << (-llh * log(2) / log(10)) << endl; - cerr << " Count: " << cnt << endl; - cerr << " OOVs: " << oovs << endl; - cerr << "Cross-entropy: " << (llh / cnt) << endl; - cerr << " Perplexity: " << pow(2, llh / cnt) << endl; - return 0; -} - - diff --git a/gi/pf/pyp_tm.cc b/gi/pf/pyp_tm.cc deleted file mode 100644 index 37b9a604..00000000 --- a/gi/pf/pyp_tm.cc +++ /dev/null @@ -1,128 +0,0 @@ -#include "pyp_tm.h" - -#include <tr1/unordered_map> -#include <iostream> -#include <queue> - -#include "tdict.h" -#include "ccrp.h" -#include "pyp_word_model.h" -#include "tied_resampler.h" - -using namespace std; -using namespace std::tr1; - -struct FreqBinner { - FreqBinner(const std::string& fname) { fd_.Load(fname); } - unsigned NumberOfBins() const { return fd_.Max() + 1; } - unsigned Bin(const WordID& w) const { return fd_.LookUp(w); } - FreqDict<unsigned> fd_; -}; - -template <typename Base, class Binner = FreqBinner> -struct ConditionalPYPWordModel { - ConditionalPYPWordModel(Base* b, const Binner* bnr = NULL) : - base(*b), - binner(bnr), - btr(binner ? binner->NumberOfBins() + 1u : 2u) {} - - void Summary() const { - cerr << "Number of conditioning contexts: " << r.size() << endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - cerr << TD::Convert(it->first) << " \tPYP(d=" << it->second.discount() << ",s=" << it->second.strength() << ") --------------------------" << endl; - for (CCRP<vector<WordID> >::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - cerr << " " << i2->second << '\t' << TD::GetString(i2->first) << endl; - } - } - - void ResampleHyperparameters(MT19937* rng) { - btr.ResampleHyperparameters(rng); - } - - prob_t Prob(const WordID src, const vector<WordID>& trglets) const { - RuleModelHash::const_iterator it = r.find(src); - if (it == r.end()) { - return base(trglets); - } else { - return it->second.prob(trglets, base(trglets)); - } - } - - void Increment(const WordID src, const vector<WordID>& trglets, MT19937* rng) { - RuleModelHash::iterator it = r.find(src); - if (it == r.end()) { - it = r.insert(make_pair(src, CCRP<vector<WordID> >(0.5,1.0))).first; - static const WordID kNULL = TD::Convert("NULL"); - unsigned bin = (src == kNULL ? 0 : 1); - if (binner && bin) { bin = binner->Bin(src) + 1; } - btr.Add(bin, &it->second); - } - if (it->second.increment(trglets, base(trglets), rng)) - base.Increment(trglets, rng); - } - - void Decrement(const WordID src, const vector<WordID>& trglets, MT19937* rng) { - RuleModelHash::iterator it = r.find(src); - assert(it != r.end()); - if (it->second.decrement(trglets, rng)) { - base.Decrement(trglets, rng); - } - } - - prob_t Likelihood() const { - prob_t p = prob_t::One(); - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - prob_t q; q.logeq(it->second.log_crp_prob()); - p *= q; - } - return p; - } - - unsigned UniqueConditioningContexts() const { - return r.size(); - } - - // TODO tie PYP hyperparameters based on source word frequency bins - Base& base; - const Binner* binner; - BinTiedResampler<CCRP<vector<WordID> > > btr; - typedef unordered_map<WordID, CCRP<vector<WordID> > > RuleModelHash; - RuleModelHash r; -}; - -PYPLexicalTranslation::PYPLexicalTranslation(const vector<vector<WordID> >& lets, - const unsigned vocab_size, - const unsigned num_letters) : - letters(lets), - base(vocab_size, num_letters, 5), - tmodel(new ConditionalPYPWordModel<PoissonUniformWordModel>(&base, new FreqBinner("10k.freq"))), - kX(-TD::Convert("X")) {} - -void PYPLexicalTranslation::Summary() const { - tmodel->Summary(); -} - -prob_t PYPLexicalTranslation::Likelihood() const { - return tmodel->Likelihood() * base.Likelihood(); -} - -void PYPLexicalTranslation::ResampleHyperparameters(MT19937* rng) { - tmodel->ResampleHyperparameters(rng); -} - -unsigned PYPLexicalTranslation::UniqueConditioningContexts() const { - return tmodel->UniqueConditioningContexts(); -} - -prob_t PYPLexicalTranslation::Prob(WordID src, WordID trg) const { - return tmodel->Prob(src, letters[trg]); -} - -void PYPLexicalTranslation::Increment(WordID src, WordID trg, MT19937* rng) { - tmodel->Increment(src, letters[trg], rng); -} - -void PYPLexicalTranslation::Decrement(WordID src, WordID trg, MT19937* rng) { - tmodel->Decrement(src, letters[trg], rng); -} - diff --git a/gi/pf/pyp_tm.h b/gi/pf/pyp_tm.h deleted file mode 100644 index 2b076a25..00000000 --- a/gi/pf/pyp_tm.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef PYP_LEX_TRANS -#define PYP_LEX_TRANS - -#include <vector> -#include "wordid.h" -#include "prob.h" -#include "sampler.h" -#include "freqdict.h" -#include "poisson_uniform_word_model.h" - -struct FreqBinner; -template <typename T, class B> struct ConditionalPYPWordModel; - -struct PYPLexicalTranslation { - explicit PYPLexicalTranslation(const std::vector<std::vector<WordID> >& lets, - const unsigned vocab_size, - const unsigned num_letters); - - prob_t Likelihood() const; - - void ResampleHyperparameters(MT19937* rng); - prob_t Prob(WordID src, WordID trg) const; // return p(trg | src) - void Summary() const; - void Increment(WordID src, WordID trg, MT19937* rng); - void Decrement(WordID src, WordID trg, MT19937* rng); - unsigned UniqueConditioningContexts() const; - - private: - const std::vector<std::vector<WordID> >& letters; // spelling dictionary - PoissonUniformWordModel base; // "generator" of English types - ConditionalPYPWordModel<PoissonUniformWordModel, FreqBinner>* tmodel; // translation distributions - // (model English word | French word) - const WordID kX; -}; - -#endif diff --git a/gi/pf/pyp_word_model.h b/gi/pf/pyp_word_model.h deleted file mode 100644 index 0bebb751..00000000 --- a/gi/pf/pyp_word_model.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef _PYP_WORD_MODEL_H_ -#define _PYP_WORD_MODEL_H_ - -#include <iostream> -#include <cmath> -#include <vector> -#include "prob.h" -#include "ccrp.h" -#include "m.h" -#include "tdict.h" -#include "os_phrase.h" - -// PYP(d,s,poisson-uniform) represented as a CRP -template <class Base> -struct PYPWordModel { - explicit PYPWordModel(Base* b) : - base(*b), - r(1,1,1,1,0.66,50.0) - {} - - void ResampleHyperparameters(MT19937* rng) { - r.resample_hyperparameters(rng); - std::cerr << " PYPWordModel(d=" << r.discount() << ",s=" << r.strength() << ")\n"; - } - - inline prob_t operator()(const std::vector<WordID>& s) const { - return r.prob(s, base(s)); - } - - inline void Increment(const std::vector<WordID>& s, MT19937* rng) { - if (r.increment(s, base(s), rng)) - base.Increment(s, rng); - } - - inline void Decrement(const std::vector<WordID>& s, MT19937 *rng) { - if (r.decrement(s, rng)) - base.Decrement(s, rng); - } - - inline prob_t Likelihood() const { - prob_t p; p.logeq(r.log_crp_prob()); - p *= base.Likelihood(); - return p; - } - - void Summary() const { - std::cerr << "PYPWordModel: generations=" << r.num_customers() - << " PYP(d=" << r.discount() << ",s=" << r.strength() << ')' << std::endl; - for (typename CCRP<std::vector<WordID> >::const_iterator it = r.begin(); it != r.end(); ++it) { - std::cerr << " " << it->second - << TD::GetString(it->first) << std::endl; - } - } - - private: - - Base& base; // keeps track of the draws from the base distribution - CCRP<std::vector<WordID> > r; -}; - -#endif diff --git a/gi/pf/quasi_model2.h b/gi/pf/quasi_model2.h deleted file mode 100644 index 4075affe..00000000 --- a/gi/pf/quasi_model2.h +++ /dev/null @@ -1,177 +0,0 @@ -#ifndef _QUASI_MODEL2_H_ -#define _QUASI_MODEL2_H_ - -#include <vector> -#include <cmath> -#include <tr1/unordered_map> -#include "boost/functional.hpp" -#include "prob.h" -#include "array2d.h" -#include "slice_sampler.h" -#include "m.h" -#include "have_64_bits.h" - -struct AlignmentObservation { - AlignmentObservation() : src_len(), trg_len(), j(), a_j() {} - AlignmentObservation(unsigned sl, unsigned tl, unsigned tw, unsigned sw) : - src_len(sl), trg_len(tl), j(tw), a_j(sw) {} - unsigned short src_len; - unsigned short trg_len; - unsigned short j; - unsigned short a_j; -}; - -#ifdef HAVE_64_BITS -inline size_t hash_value(const AlignmentObservation& o) { - return reinterpret_cast<const size_t&>(o); -} -inline bool operator==(const AlignmentObservation& a, const AlignmentObservation& b) { - return hash_value(a) == hash_value(b); -} -#else -inline size_t hash_value(const AlignmentObservation& o) { - size_t h = 1; - boost::hash_combine(h, o.src_len); - boost::hash_combine(h, o.trg_len); - boost::hash_combine(h, o.j); - boost::hash_combine(h, o.a_j); - return h; -} -#endif - -struct QuasiModel2 { - explicit QuasiModel2(double alpha, double pnull = 0.1) : - alpha_(alpha), - pnull_(pnull), - pnotnull_(1 - pnull) {} - - // a_j = 0 => NULL; src_len does *not* include null - prob_t Prob(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len) const { - if (!a_j) return pnull_; - return pnotnull_ * - prob_t(UnnormalizedProb(a_j, j, src_len, trg_len, alpha_) / GetOrComputeZ(j, src_len, trg_len)); - } - - void Increment(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len) { - assert(a_j <= src_len); - assert(j < trg_len); - ++obs_[AlignmentObservation(src_len, trg_len, j, a_j)]; - } - - void Decrement(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len) { - const AlignmentObservation ao(src_len, trg_len, j, a_j); - int &cc = obs_[ao]; - assert(cc > 0); - --cc; - if (!cc) obs_.erase(ao); - } - - struct PNullResampler { - PNullResampler(const QuasiModel2& m) : m_(m) {} - const QuasiModel2& m_; - double operator()(const double& proposed_pnull) const { - return log(m_.Likelihood(m_.alpha_, proposed_pnull)); - } - }; - - struct AlphaResampler { - AlphaResampler(const QuasiModel2& m) : m_(m) {} - const QuasiModel2& m_; - double operator()(const double& proposed_alpha) const { - return log(m_.Likelihood(proposed_alpha, m_.pnull_.as_float())); - } - }; - - void ResampleHyperparameters(MT19937* rng, const unsigned nloop = 5, const unsigned niterations = 10) { - const PNullResampler dr(*this); - const AlphaResampler ar(*this); - for (unsigned i = 0; i < nloop; ++i) { - double pnull = slice_sampler1d(dr, pnull_.as_float(), *rng, 0.00000001, - 1.0, 0.0, niterations, 100*niterations); - pnull_ = prob_t(pnull); - alpha_ = slice_sampler1d(ar, alpha_, *rng, 0.00000001, - std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); - } - std::cerr << "QuasiModel2(alpha=" << alpha_ << ",p_null=" - << pnull_.as_float() << ") = " << Likelihood() << std::endl; - zcache_.clear(); - } - - prob_t Likelihood() const { - return Likelihood(alpha_, pnull_.as_float()); - } - - prob_t Likelihood(double alpha, double ppnull) const { - const prob_t pnull(ppnull); - const prob_t pnotnull(1 - ppnull); - - prob_t p; - p.logeq(Md::log_gamma_density(alpha, 0.1, 25)); // TODO configure - assert(!p.is_0()); - prob_t prob_of_ppnull; prob_of_ppnull.logeq(Md::log_beta_density(ppnull, 2, 10)); - assert(!prob_of_ppnull.is_0()); - p *= prob_of_ppnull; - for (ObsCount::const_iterator it = obs_.begin(); it != obs_.end(); ++it) { - const AlignmentObservation& ao = it->first; - if (ao.a_j) { - prob_t u = XUnnormalizedProb(ao.a_j, ao.j, ao.src_len, ao.trg_len, alpha); - prob_t z = XComputeZ(ao.j, ao.src_len, ao.trg_len, alpha); - prob_t pa(u / z); - pa *= pnotnull; - pa.poweq(it->second); - p *= pa; - } else { - p *= pnull.pow(it->second); - } - } - return p; - } - - private: - static prob_t XUnnormalizedProb(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len, double alpha) { - prob_t p; - p.logeq(-fabs(double(a_j - 1) / src_len - double(j) / trg_len) * alpha); - return p; - } - - static prob_t XComputeZ(unsigned j, unsigned src_len, unsigned trg_len, double alpha) { - prob_t z = prob_t::Zero(); - for (int a_j = 1; a_j <= src_len; ++a_j) - z += XUnnormalizedProb(a_j, j, src_len, trg_len, alpha); - return z; - } - - static double UnnormalizedProb(unsigned a_j, unsigned j, unsigned src_len, unsigned trg_len, double alpha) { - return exp(-fabs(double(a_j - 1) / src_len - double(j) / trg_len) * alpha); - } - - static double ComputeZ(unsigned j, unsigned src_len, unsigned trg_len, double alpha) { - double z = 0; - for (int a_j = 1; a_j <= src_len; ++a_j) - z += UnnormalizedProb(a_j, j, src_len, trg_len, alpha); - return z; - } - - const double& GetOrComputeZ(unsigned j, unsigned src_len, unsigned trg_len) const { - if (src_len >= zcache_.size()) - zcache_.resize(src_len + 1); - if (trg_len >= zcache_[src_len].size()) - zcache_[src_len].resize(trg_len + 1); - std::vector<double>& zv = zcache_[src_len][trg_len]; - if (zv.size() == 0) - zv.resize(trg_len); - double& z = zv[j]; - if (!z) - z = ComputeZ(j, src_len, trg_len, alpha_); - return z; - } - - double alpha_; - prob_t pnull_; - prob_t pnotnull_; - mutable std::vector<std::vector<std::vector<double> > > zcache_; - typedef std::tr1::unordered_map<AlignmentObservation, int, boost::hash<AlignmentObservation> > ObsCount; - ObsCount obs_; -}; - -#endif diff --git a/gi/pf/reachability.cc b/gi/pf/reachability.cc deleted file mode 100644 index 7d0d04ac..00000000 --- a/gi/pf/reachability.cc +++ /dev/null @@ -1,74 +0,0 @@ -#include "reachability.h" - -#include <vector> -#include <iostream> - -using namespace std; - -struct SState { - SState() : prev_src_covered(), prev_trg_covered() {} - SState(int i, int j) : prev_src_covered(i), prev_trg_covered(j) {} - int prev_src_covered; - int prev_trg_covered; -}; - -void Reachability::ComputeReachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) { - typedef boost::multi_array<vector<SState>, 2> array_type; - array_type a(boost::extents[srclen + 1][trglen + 1]); - a[0][0].push_back(SState()); - for (int i = 0; i < srclen; ++i) { - for (int j = 0; j < trglen; ++j) { - if (a[i][j].size() == 0) continue; - const SState prev(i,j); - for (int k = 1; k <= src_max_phrase_len; ++k) { - if ((i + k) > srclen) continue; - for (int l = 1; l <= trg_max_phrase_len; ++l) { - if ((j + l) > trglen) continue; - a[i + k][j + l].push_back(prev); - } - } - } - } - a[0][0].clear(); - //cerr << srclen << "," << trglen << ": Final cell contains " << a[srclen][trglen].size() << " back pointers\n"; - if (a[srclen][trglen].empty()) { - cerr << "Sequence pair with lengths (" << srclen << ',' << trglen << ") violates reachability constraints\n"; - nodes = 0; - return; - } - - typedef boost::multi_array<bool, 2> rarray_type; - rarray_type r(boost::extents[srclen + 1][trglen + 1]); - r[srclen][trglen] = true; - nodes = 0; - for (int i = srclen; i >= 0; --i) { - for (int j = trglen; j >= 0; --j) { - vector<SState>& prevs = a[i][j]; - if (!r[i][j]) { prevs.clear(); } - for (int k = 0; k < prevs.size(); ++k) { - r[prevs[k].prev_src_covered][prevs[k].prev_trg_covered] = true; - int src_delta = i - prevs[k].prev_src_covered; - edges[prevs[k].prev_src_covered][prevs[k].prev_trg_covered][src_delta][j - prevs[k].prev_trg_covered] = true; - valid_deltas[prevs[k].prev_src_covered][prevs[k].prev_trg_covered].push_back(make_pair<short,short>(src_delta,j - prevs[k].prev_trg_covered)); - short &msd = max_src_delta[prevs[k].prev_src_covered][prevs[k].prev_trg_covered]; - if (src_delta > msd) msd = src_delta; - } - } - } - assert(!edges[0][0][1][0]); - assert(!edges[0][0][0][1]); - assert(!edges[0][0][0][0]); - assert(max_src_delta[0][0] > 0); - nodes = 0; - for (int i = 0; i < srclen; ++i) { - for (int j = 0; j < trglen; ++j) { - if (valid_deltas[i][j].size() > 0) { - node_addresses[i][j] = nodes++; - } else { - node_addresses[i][j] = -1; - } - } - } - cerr << "Sequence pair with lengths (" << srclen << ',' << trglen << ") has " << valid_deltas[0][0].size() << " out edges in its root node, " << nodes << " nodes in total, and outside estimate matrix will require " << sizeof(float)*nodes << " bytes\n"; - } - diff --git a/gi/pf/reachability.h b/gi/pf/reachability.h deleted file mode 100644 index 1e22c76a..00000000 --- a/gi/pf/reachability.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef _REACHABILITY_H_ -#define _REACHABILITY_H_ - -#include "boost/multi_array.hpp" - -// determines minimum and maximum lengths of outgoing edges from all -// coverage positions such that the alignment path respects src and -// trg maximum phrase sizes -// -// runs in O(n^2 * src_max * trg_max) time but should be relatively fast -// -// currently forbids 0 -> n and n -> 0 alignments - -struct Reachability { - unsigned nodes; - boost::multi_array<bool, 4> edges; // edges[src_covered][trg_covered][src_delta][trg_delta] is this edge worth exploring? - boost::multi_array<short, 2> max_src_delta; // msd[src_covered][trg_covered] -- the largest src delta that's valid - boost::multi_array<short, 2> node_addresses; // na[src_covered][trg_covered] -- the index of the node in a one-dimensional array (of size "nodes") - boost::multi_array<std::vector<std::pair<short,short> >, 2> valid_deltas; // valid_deltas[src_covered][trg_covered] list of valid transitions leaving a particular node - - Reachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len) : - nodes(), - edges(boost::extents[srclen][trglen][src_max_phrase_len+1][trg_max_phrase_len+1]), - max_src_delta(boost::extents[srclen][trglen]), - node_addresses(boost::extents[srclen][trglen]), - valid_deltas(boost::extents[srclen][trglen]) { - ComputeReachability(srclen, trglen, src_max_phrase_len, trg_max_phrase_len); - } - - private: - void ComputeReachability(int srclen, int trglen, int src_max_phrase_len, int trg_max_phrase_len); -}; - -#endif diff --git a/gi/pf/tied_resampler.h b/gi/pf/tied_resampler.h deleted file mode 100644 index a4f4af36..00000000 --- a/gi/pf/tied_resampler.h +++ /dev/null @@ -1,122 +0,0 @@ -#ifndef _TIED_RESAMPLER_H_ -#define _TIED_RESAMPLER_H_ - -#include <set> -#include <vector> -#include "sampler.h" -#include "slice_sampler.h" -#include "m.h" - -template <class CRP> -struct TiedResampler { - explicit TiedResampler(double da, double db, double ss, double sr, double d=0.5, double s=1.0) : - d_alpha(da), - d_beta(db), - s_shape(ss), - s_rate(sr), - discount(d), - strength(s) {} - - void Add(CRP* crp) { - crps.insert(crp); - crp->set_discount(discount); - crp->set_strength(strength); - assert(!crp->has_discount_prior()); - assert(!crp->has_strength_prior()); - } - - void Remove(CRP* crp) { - crps.erase(crp); - } - - size_t size() const { - return crps.size(); - } - - double LogLikelihood(double d, double s) const { - if (s <= -d) return -std::numeric_limits<double>::infinity(); - double llh = Md::log_beta_density(d, d_alpha, d_beta) + - Md::log_gamma_density(d + s, s_shape, s_rate); - for (typename std::set<CRP*>::iterator it = crps.begin(); it != crps.end(); ++it) - llh += (*it)->log_crp_prob(d, s); - return llh; - } - - double LogLikelihood() const { - return LogLikelihood(discount, strength); - } - - struct DiscountResampler { - DiscountResampler(const TiedResampler& m) : m_(m) {} - const TiedResampler& m_; - double operator()(const double& proposed_discount) const { - return m_.LogLikelihood(proposed_discount, m_.strength); - } - }; - - struct AlphaResampler { - AlphaResampler(const TiedResampler& m) : m_(m) {} - const TiedResampler& m_; - double operator()(const double& proposed_strength) const { - return m_.LogLikelihood(m_.discount, proposed_strength); - } - }; - - void ResampleHyperparameters(MT19937* rng, const unsigned nloop = 5, const unsigned niterations = 10) { - if (size() == 0) { std::cerr << "EMPTY - not resampling\n"; return; } - const DiscountResampler dr(*this); - const AlphaResampler ar(*this); - for (int iter = 0; iter < nloop; ++iter) { - strength = slice_sampler1d(ar, strength, *rng, -discount + std::numeric_limits<double>::min(), - std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); - double min_discount = std::numeric_limits<double>::min(); - if (strength < 0.0) min_discount -= strength; - discount = slice_sampler1d(dr, discount, *rng, min_discount, - 1.0, 0.0, niterations, 100*niterations); - } - strength = slice_sampler1d(ar, strength, *rng, -discount + std::numeric_limits<double>::min(), - std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); - std::cerr << "TiedCRPs(d=" << discount << ",s=" - << strength << ") = " << LogLikelihood(discount, strength) << std::endl; - for (typename std::set<CRP*>::iterator it = crps.begin(); it != crps.end(); ++it) - (*it)->set_hyperparameters(discount, strength); - } - private: - std::set<CRP*> crps; - const double d_alpha, d_beta, s_shape, s_rate; - double discount, strength; -}; - -// split according to some criterion -template <class CRP> -struct BinTiedResampler { - explicit BinTiedResampler(unsigned nbins) : - resamplers(nbins, TiedResampler<CRP>(1,1,1,1)) {} - - void Add(unsigned bin, CRP* crp) { - resamplers[bin].Add(crp); - } - - void Remove(unsigned bin, CRP* crp) { - resamplers[bin].Remove(crp); - } - - void ResampleHyperparameters(MT19937* rng) { - for (unsigned i = 0; i < resamplers.size(); ++i) { - std::cerr << "BIN " << i << " (" << resamplers[i].size() << " CRPs): " << std::flush; - resamplers[i].ResampleHyperparameters(rng); - } - } - - double LogLikelihood() const { - double llh = 0; - for (unsigned i = 0; i < resamplers.size(); ++i) - llh += resamplers[i].LogLikelihood(); - return llh; - } - - private: - std::vector<TiedResampler<CRP> > resamplers; -}; - -#endif diff --git a/gi/pf/tpf.cc b/gi/pf/tpf.cc deleted file mode 100644 index 7348d21c..00000000 --- a/gi/pf/tpf.cc +++ /dev/null @@ -1,99 +0,0 @@ -#include <iostream> -#include <tr1/memory> -#include <queue> - -#include "sampler.h" - -using namespace std; -using namespace tr1; - -shared_ptr<MT19937> prng; - -struct Particle { - Particle() : weight(prob_t::One()) {} - vector<int> states; - prob_t weight; - prob_t gamma_last; -}; - -ostream& operator<<(ostream& os, const Particle& p) { - os << "["; - for (int i = 0; i < p.states.size(); ++i) os << p.states[i] << ' '; - os << "| w=" << log(p.weight) << ']'; - return os; -} - -void Rejuvenate(vector<Particle>& pps) { - SampleSet<prob_t> ss; - vector<Particle> nps(pps.size()); - for (int i = 0; i < pps.size(); ++i) { -// cerr << pps[i] << endl; - ss.add(pps[i].weight); - } -// cerr << "REJUVINATING...\n"; - for (int i = 0; i < pps.size(); ++i) { - nps[i] = pps[prng->SelectSample(ss)]; - nps[i].weight = prob_t(1.0 / pps.size()); -// cerr << nps[i] << endl; - } - nps.swap(pps); -// exit(1); -} - -int main(int argc, char** argv) { - const unsigned particles = 100; - prng.reset(new MT19937); - MT19937& rng = *prng; - - // q(a) = 0.8 - // q(b) = 0.8 - // q(c) = 0.4 - SampleSet<double> ssq; - ssq.add(0.4); - ssq.add(0.6); - ssq.add(0); - double qz = 1; - - // p(a) = 0.2 - // p(b) = 0.8 - vector<double> p(3); - p[0] = 0.2; - p[1] = 0.8; - p[2] = 0; - - vector<int> counts(3); - int tot = 0; - - vector<Particle> pps(particles); - SampleSet<prob_t> ppss; - int LEN = 12; - int PP = 1; - while (pps[0].states.size() < LEN) { - for (int pi = 0; pi < particles; ++pi) { - Particle& prt = pps[pi]; - - bool redo = true; - const Particle savedp = prt; - while (redo) { - redo = false; - for (int i = 0; i < PP; ++i) { - int s = rng.SelectSample(ssq); - double gamma_last = p[s]; - if (!gamma_last) { redo = true; break; } - double q = ssq[s] / qz; - prt.states.push_back(s); - prt.weight *= prob_t(gamma_last / q); - } - if (redo) { prt = savedp; continue; } - } - } - Rejuvenate(pps); - } - ppss.clear(); - for (int i = 0; i < particles; ++i) { ppss.add(pps[i].weight); } - int sp = rng.SelectSample(ppss); - cerr << pps[sp] << endl; - - return 0; -} - diff --git a/gi/pf/transliterations.cc b/gi/pf/transliterations.cc deleted file mode 100644 index b2996f65..00000000 --- a/gi/pf/transliterations.cc +++ /dev/null @@ -1,334 +0,0 @@ -#include "transliterations.h" - -#include <iostream> -#include <vector> - -#include "boost/shared_ptr.hpp" - -#include "backward.h" -#include "filelib.h" -#include "tdict.h" -#include "trule.h" -#include "filelib.h" -#include "ccrp_nt.h" -#include "m.h" -#include "reachability.h" - -using namespace std; -using namespace std::tr1; - -struct TruncatedConditionalLengthModel { - TruncatedConditionalLengthModel(unsigned max_src_size, unsigned max_trg_size, double expected_src_to_trg_ratio) : - plens(max_src_size+1, vector<prob_t>(max_trg_size+1, 0.0)) { - for (unsigned i = 1; i <= max_src_size; ++i) { - prob_t z = prob_t::Zero(); - for (unsigned j = 1; j <= max_trg_size; ++j) - z += (plens[i][j] = prob_t(0.01 + exp(Md::log_poisson(j, i * expected_src_to_trg_ratio)))); - for (unsigned j = 1; j <= max_trg_size; ++j) - plens[i][j] /= z; - //for (unsigned j = 1; j <= max_trg_size; ++j) - // cerr << "P(trg_len=" << j << " | src_len=" << i << ") = " << plens[i][j] << endl; - } - } - - // return p(tlen | slen) for *chunks* not full words - inline const prob_t& operator()(int slen, int tlen) const { - return plens[slen][tlen]; - } - - vector<vector<prob_t> > plens; -}; - -struct CondBaseDist { - CondBaseDist(unsigned max_src_size, unsigned max_trg_size, double expected_src_to_trg_ratio) : - tclm(max_src_size, max_trg_size, expected_src_to_trg_ratio) {} - - prob_t operator()(const vector<WordID>& src, unsigned sf, unsigned st, - const vector<WordID>& trg, unsigned tf, unsigned tt) const { - prob_t p = tclm(st - sf, tt - tf); // target len | source length ~ TCLM(source len) - assert(!"not impl"); - return p; - } - inline prob_t operator()(const vector<WordID>& src, const vector<WordID>& trg) const { - return (*this)(src, 0, src.size(), trg, 0, trg.size()); - } - TruncatedConditionalLengthModel tclm; -}; - -// represents transliteration phrase probabilities, e.g. -// p( a l - | A l ) , p( o | A w ) , ... -struct TransliterationChunkConditionalModel { - explicit TransliterationChunkConditionalModel(const CondBaseDist& pp0) : - d(0.0), - strength(1.0), - rp0(pp0) { - } - - void Summary() const { - std::cerr << "Number of conditioning contexts: " << r.size() << std::endl; - for (RuleModelHash::const_iterator it = r.begin(); it != r.end(); ++it) { - std::cerr << TD::GetString(it->first) << " \t(\\alpha = " << it->second.alpha() << ") --------------------------" << std::endl; - for (CCRP_NoTable<TRule>::const_iterator i2 = it->second.begin(); i2 != it->second.end(); ++i2) - std::cerr << " " << i2->second << '\t' << i2->first << std::endl; - } - } - - int DecrementRule(const TRule& rule) { - RuleModelHash::iterator it = r.find(rule.f_); - assert(it != r.end()); - int count = it->second.decrement(rule); - if (count) { - if (it->second.num_customers() == 0) r.erase(it); - } - return count; - } - - int IncrementRule(const TRule& rule) { - RuleModelHash::iterator it = r.find(rule.f_); - if (it == r.end()) { - it = r.insert(make_pair(rule.f_, CCRP_NoTable<TRule>(strength))).first; - } - int count = it->second.increment(rule); - return count; - } - - void IncrementRules(const std::vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - IncrementRule(*rules[i]); - } - - void DecrementRules(const std::vector<TRulePtr>& rules) { - for (int i = 0; i < rules.size(); ++i) - DecrementRule(*rules[i]); - } - - prob_t RuleProbability(const TRule& rule) const { - prob_t p; - RuleModelHash::const_iterator it = r.find(rule.f_); - if (it == r.end()) { - p = rp0(rule.f_, rule.e_); - } else { - p = it->second.prob(rule, rp0(rule.f_, rule.e_)); - } - return p; - } - - double LogLikelihood(const double& dd, const double& aa) const { - if (aa <= -dd) return -std::numeric_limits<double>::infinity(); - //double llh = Md::log_beta_density(dd, 10, 3) + Md::log_gamma_density(aa, 1, 1); - double llh = //Md::log_beta_density(dd, 1, 1) + - Md::log_gamma_density(dd + aa, 1, 1); - std::tr1::unordered_map<std::vector<WordID>, CCRP_NoTable<TRule>, boost::hash<std::vector<WordID> > >::const_iterator it; - for (it = r.begin(); it != r.end(); ++it) - llh += it->second.log_crp_prob(aa); - return llh; - } - - struct AlphaResampler { - AlphaResampler(const TransliterationChunkConditionalModel& m) : m_(m) {} - const TransliterationChunkConditionalModel& m_; - double operator()(const double& proposed_strength) const { - return m_.LogLikelihood(m_.d, proposed_strength); - } - }; - - void ResampleHyperparameters(MT19937* rng) { - std::tr1::unordered_map<std::vector<WordID>, CCRP_NoTable<TRule>, boost::hash<std::vector<WordID> > >::iterator it; - //const unsigned nloop = 5; - const unsigned niterations = 10; - //DiscountResampler dr(*this); - AlphaResampler ar(*this); -#if 0 - for (int iter = 0; iter < nloop; ++iter) { - strength = slice_sampler1d(ar, strength, *rng, -d + std::numeric_limits<double>::min(), - std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); - double min_discount = std::numeric_limits<double>::min(); - if (strength < 0.0) min_discount -= strength; - d = slice_sampler1d(dr, d, *rng, min_discount, - 1.0, 0.0, niterations, 100*niterations); - } -#endif - strength = slice_sampler1d(ar, strength, *rng, -d, - std::numeric_limits<double>::infinity(), 0.0, niterations, 100*niterations); - std::cerr << "CTMModel(alpha=" << strength << ") = " << LogLikelihood(d, strength) << std::endl; - for (it = r.begin(); it != r.end(); ++it) { -#if 0 - it->second.set_discount(d); -#endif - it->second.set_alpha(strength); - } - } - - prob_t Likelihood() const { - prob_t p; p.logeq(LogLikelihood(d, strength)); - return p; - } - - const CondBaseDist& rp0; - typedef std::tr1::unordered_map<std::vector<WordID>, - CCRP_NoTable<TRule>, - boost::hash<std::vector<WordID> > > RuleModelHash; - RuleModelHash r; - double d, strength; -}; - -struct GraphStructure { - GraphStructure() : r() {} - // leak memory - these are basically static - const Reachability* r; - bool IsReachable() const { return r->nodes > 0; } -}; - -struct ProbabilityEstimates { - ProbabilityEstimates() : gs(), backward() {} - explicit ProbabilityEstimates(const GraphStructure& g) : - gs(&g), backward() { - if (g.r->nodes > 0) - backward = new float[g.r->nodes]; - } - // leak memory, these are static - - // returns an estimate of the marginal probability - double MarginalEstimate() const { - if (!backward) return 0; - return backward[0]; - } - - // returns an backward estimate - double Backward(int src_covered, int trg_covered) const { - if (!backward) return 0; - int ind = gs->r->node_addresses[src_covered][trg_covered]; - if (ind < 0) return 0; - return backward[ind]; - } - - prob_t estp; - float* backward; - private: - const GraphStructure* gs; -}; - -struct TransliterationsImpl { - TransliterationsImpl(int max_src, int max_trg, double sr, const BackwardEstimator& b) : - cp0(max_src, max_trg, sr), - tccm(cp0), - be(b), - kMAX_SRC_CHUNK(max_src), - kMAX_TRG_CHUNK(max_trg), - kS2T_RATIO(sr), - tot_pairs(), tot_mem() { - } - const CondBaseDist cp0; - TransliterationChunkConditionalModel tccm; - const BackwardEstimator& be; - - void Initialize(WordID src, const vector<WordID>& src_lets, WordID trg, const vector<WordID>& trg_lets) { - const size_t src_len = src_lets.size(); - const size_t trg_len = trg_lets.size(); - - // init graph structure - if (src_len >= graphs.size()) graphs.resize(src_len + 1); - if (trg_len >= graphs[src_len].size()) graphs[src_len].resize(trg_len + 1); - GraphStructure& gs = graphs[src_len][trg_len]; - if (!gs.r) { - double rat = exp(fabs(log(trg_len / (src_len * kS2T_RATIO)))); - if (rat > 1.5 || (rat > 2.4 && src_len < 6)) { - cerr << " ** Forbidding transliterations of size " << src_len << "," << trg_len << ": " << rat << endl; - gs.r = new Reachability(src_len, trg_len, 0, 0); - } else { - gs.r = new Reachability(src_len, trg_len, kMAX_SRC_CHUNK, kMAX_TRG_CHUNK); - } - } - - const Reachability& r = *gs.r; - - // init backward estimates - if (src >= ests.size()) ests.resize(src + 1); - unordered_map<WordID, ProbabilityEstimates>::iterator it = ests[src].find(trg); - if (it != ests[src].end()) return; // already initialized - - it = ests[src].insert(make_pair(trg, ProbabilityEstimates(gs))).first; - ProbabilityEstimates& est = it->second; - if (!gs.r->nodes) return; // not derivable subject to length constraints - - be.InitializeGrid(src_lets, trg_lets, r, kS2T_RATIO, est.backward); - cerr << TD::GetString(src_lets) << " ||| " << TD::GetString(trg_lets) << " ||| " << (est.backward[0] / trg_lets.size()) << endl; - tot_pairs++; - tot_mem += sizeof(float) * gs.r->nodes; - } - - void Forbid(WordID src, const vector<WordID>& src_lets, WordID trg, const vector<WordID>& trg_lets) { - const size_t src_len = src_lets.size(); - const size_t trg_len = trg_lets.size(); - // TODO - } - - prob_t EstimateProbability(WordID s, const vector<WordID>& src, WordID t, const vector<WordID>& trg) const { - assert(src.size() < graphs.size()); - const vector<GraphStructure>& tv = graphs[src.size()]; - assert(trg.size() < tv.size()); - const GraphStructure& gs = tv[trg.size()]; - if (gs.r->nodes == 0) - return prob_t::Zero(); - const unordered_map<WordID, ProbabilityEstimates>::const_iterator it = ests[s].find(t); - assert(it != ests[s].end()); - return it->second.estp; - } - - void GraphSummary() const { - double to = 0; - double tn = 0; - double tt = 0; - for (int i = 0; i < graphs.size(); ++i) { - const vector<GraphStructure>& vt = graphs[i]; - for (int j = 0; j < vt.size(); ++j) { - const GraphStructure& gs = vt[j]; - if (!gs.r) continue; - tt++; - for (int k = 0; k < i; ++k) { - for (int l = 0; l < j; ++l) { - size_t c = gs.r->valid_deltas[k][l].size(); - if (c) { - tn += 1; - to += c; - } - } - } - } - } - cerr << " Average nodes = " << (tn / tt) << endl; - cerr << "Average out-degree = " << (to / tn) << endl; - cerr << " Unique structures = " << tt << endl; - cerr << " Unique pairs = " << tot_pairs << endl; - cerr << " BEs size = " << (tot_mem / (1024.0*1024.0)) << " MB" << endl; - } - - const int kMAX_SRC_CHUNK; - const int kMAX_TRG_CHUNK; - const double kS2T_RATIO; - unsigned tot_pairs; - size_t tot_mem; - vector<vector<GraphStructure> > graphs; // graphs[src_len][trg_len] - vector<unordered_map<WordID, ProbabilityEstimates> > ests; // ests[src][trg] -}; - -Transliterations::Transliterations(int max_src, int max_trg, double sr, const BackwardEstimator& be) : - pimpl_(new TransliterationsImpl(max_src, max_trg, sr, be)) {} -Transliterations::~Transliterations() { delete pimpl_; } - -void Transliterations::Initialize(WordID src, const vector<WordID>& src_lets, WordID trg, const vector<WordID>& trg_lets) { - pimpl_->Initialize(src, src_lets, trg, trg_lets); -} - -prob_t Transliterations::EstimateProbability(WordID s, const vector<WordID>& src, WordID t, const vector<WordID>& trg) const { - return pimpl_->EstimateProbability(s, src,t, trg); -} - -void Transliterations::Forbid(WordID src, const vector<WordID>& src_lets, WordID trg, const vector<WordID>& trg_lets) { - pimpl_->Forbid(src, src_lets, trg, trg_lets); -} - -void Transliterations::GraphSummary() const { - pimpl_->GraphSummary(); -} - diff --git a/gi/pf/transliterations.h b/gi/pf/transliterations.h deleted file mode 100644 index 49d14684..00000000 --- a/gi/pf/transliterations.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef _TRANSLITERATIONS_H_ -#define _TRANSLITERATIONS_H_ - -#include <vector> -#include "wordid.h" -#include "prob.h" - -struct BackwardEstimator; -struct TransliterationsImpl; -struct Transliterations { - // max_src and max_trg indicate how big the transliteration phrases can be - // see reachability.h for information about filter_ratio - explicit Transliterations(int max_src, int max_trg, double s2t_rat, const BackwardEstimator& be); - ~Transliterations(); - void Initialize(WordID src, const std::vector<WordID>& src_lets, WordID trg, const std::vector<WordID>& trg_lets); - void Forbid(WordID src, const std::vector<WordID>& src_lets, WordID trg, const std::vector<WordID>& trg_lets); - void GraphSummary() const; - prob_t EstimateProbability(WordID s, const std::vector<WordID>& src, WordID t, const std::vector<WordID>& trg) const; - private: - TransliterationsImpl* pimpl_; -}; - -#endif - diff --git a/gi/pf/unigrams.cc b/gi/pf/unigrams.cc deleted file mode 100644 index 40829775..00000000 --- a/gi/pf/unigrams.cc +++ /dev/null @@ -1,80 +0,0 @@ -#include "unigrams.h" - -#include <string> -#include <cmath> - -#include "stringlib.h" -#include "filelib.h" - -using namespace std; - -void UnigramModel::LoadUnigrams(const string& fname) { - cerr << "Loading unigram probabilities from " << fname << " ..." << endl; - ReadFile rf(fname); - string line; - istream& in = *rf.stream(); - assert(in); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\data\\"); - getline(in, line); - size_t pos = line.find("ngram 1="); - assert(pos == 0); - assert(line.size() > 8); - const size_t num_unigrams = atoi(&line[8]); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\1-grams:"); - for (size_t i = 0; i < num_unigrams; ++i) { - getline(in, line); - assert(line.size() > 0); - pos = line.find('\t'); - assert(pos > 0); - assert(pos + 1 < line.size()); - const WordID w = TD::Convert(line.substr(pos + 1)); - line[pos] = 0; - float p = atof(&line[0]); - if (w < probs_.size()) probs_[w].logeq(p * log(10)); else cerr << "WARNING: don't know about '" << TD::Convert(w) << "'\n"; - } -} - -void UnigramWordModel::LoadUnigrams(const string& fname) { - cerr << "Loading unigram probabilities from " << fname << " ..." << endl; - ReadFile rf(fname); - string line; - istream& in = *rf.stream(); - assert(in); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\data\\"); - getline(in, line); - size_t pos = line.find("ngram 1="); - assert(pos == 0); - assert(line.size() > 8); - const size_t num_unigrams = atoi(&line[8]); - getline(in, line); - assert(line.empty()); - getline(in, line); - assert(line == "\\1-grams:"); - for (size_t i = 0; i < num_unigrams; ++i) { - getline(in, line); - assert(line.size() > 0); - pos = line.find('\t'); - assert(pos > 0); - assert(pos + 1 < line.size()); - size_t cur = pos + 1; - vector<WordID> w; - while (cur < line.size()) { - const size_t len = UTF8Len(line[cur]); - w.push_back(TD::Convert(line.substr(cur, len))); - cur += len; - } - line[pos] = 0; - float p = atof(&line[0]); - probs_[w].logeq(p * log(10.0)); - } -} - diff --git a/gi/pf/unigrams.h b/gi/pf/unigrams.h deleted file mode 100644 index 1660d1ed..00000000 --- a/gi/pf/unigrams.h +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef _UNIGRAMS_H_ -#define _UNIGRAMS_H_ - -#include <vector> -#include <string> -#include <tr1/unordered_map> -#include <boost/functional.hpp> - -#include "wordid.h" -#include "prob.h" -#include "tdict.h" - -struct UnigramModel { - explicit UnigramModel(const std::string& fname, unsigned vocab_size) : - use_uniform_(fname.size() == 0), - uniform_(1.0 / vocab_size), - probs_() { - if (fname.size() > 0) { - probs_.resize(TD::NumWords() + 1); - LoadUnigrams(fname); - } - } - - const prob_t& operator()(const WordID& w) const { - assert(w); - if (use_uniform_) return uniform_; - return probs_[w]; - } - - private: - void LoadUnigrams(const std::string& fname); - - const bool use_uniform_; - const prob_t uniform_; - std::vector<prob_t> probs_; -}; - - -// reads an ARPA unigram file and converts words like 'cat' into a string 'c a t' -struct UnigramWordModel { - explicit UnigramWordModel(const std::string& fname) : - use_uniform_(false), - uniform_(1.0), - probs_() { - LoadUnigrams(fname); - } - - explicit UnigramWordModel(const unsigned vocab_size) : - use_uniform_(true), - uniform_(1.0 / vocab_size), - probs_() {} - - const prob_t& operator()(const std::vector<WordID>& s) const { - if (use_uniform_) return uniform_; - const VectorProbHash::const_iterator it = probs_.find(s); - assert(it != probs_.end()); - return it->second; - } - - private: - void LoadUnigrams(const std::string& fname); - - const bool use_uniform_; - const prob_t uniform_; - typedef std::tr1::unordered_map<std::vector<WordID>, prob_t, boost::hash<std::vector<WordID> > > VectorProbHash; - VectorProbHash probs_; -}; - -#endif diff --git a/gi/pipeline/OLD.clsp.config b/gi/pipeline/OLD.clsp.config deleted file mode 100644 index cd0f9d65..00000000 --- a/gi/pipeline/OLD.clsp.config +++ /dev/null @@ -1,9 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM xfeats.grammar dev dev-refs test1 testt-eval.sh ... -btec /export/ws10smt/data/btec/ split.zh-en.al lm/en.3gram.lm.gz xgrammar/grammar.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -fbis /export/ws10smt/data/chinese-english.fbis corpus.zh-en.al -zhen /export/ws10smt/data/chinese-english corpus.zh-en.al -aren /export/ws10smt/data/arabic-english corpus.ar-en.al -uren /export/ws10smt/data/urdu-english corpus.ur-en.al -nlfr /export/ws10smt/data/dutch-french corpus.nl-fr.al - diff --git a/gi/pipeline/OLD.evaluation-pipeline.pl b/gi/pipeline/OLD.evaluation-pipeline.pl deleted file mode 100755 index 49c303eb..00000000 --- a/gi/pipeline/OLD.evaluation-pipeline.pl +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use Getopt::Long; -use Cwd; -my $CWD = getcwd; - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } - -my @DEFAULT_FEATS = qw( - LogRuleCount SingletonRule LexE2F LexF2E WordPenalty - LogFCount LanguageModel Glue GlueTop PassThrough SingletonF -); - -my %init_weights = qw( - LogRuleCount 0.2 - LexE2F -0.3 - LexF2E -0.3 - LogFCount 0.1 - WordPenalty -1.5 - LanguageModel 1.2 - Glue -1.0 - GlueTop 0.00001 - PassThrough -10.0 - SingletonRule -0.1 - X_EGivenF -0.3 - X_FGivenE -0.3 - X_LogECount -1 - X_LogFCount -0.1 - X_LogRuleCount 0.3 - X_SingletonE -0.1 - X_SingletonF -0.1 - X_SingletonRule -0.5 -); - -my $CDEC = "$SCRIPT_DIR/../../decoder/cdec"; -my $PARALLELIZE = "$SCRIPT_DIR/../../vest/parallelize.pl"; -my $EXTOOLS = "$SCRIPT_DIR/../../extools"; -die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS; -my $VEST = "$SCRIPT_DIR/../../vest"; -die "Can't find vest: $VEST" unless -e $VEST && -d $VEST; -my $DISTVEST = "$VEST/dist-vest.pl"; -my $FILTSCORE = "$EXTOOLS/filter_score_grammar"; -my $ADDXFEATS = "$SCRIPT_DIR/scripts/xfeats.pl"; -assert_exec($CDEC, $PARALLELIZE, $FILTSCORE, $DISTVEST, $ADDXFEATS); - -my $config = "$SCRIPT_DIR/OLD.clsp.config"; -print STDERR "CORPORA CONFIGURATION: $config\n"; -open CONF, "<$config" or die "Can't read $config: $!"; -my %paths; -my %corpora; -my %lms; -my %devs; -my %devrefs; -my %tests; -my %testevals; -my %xgrammars; -print STDERR " LANGUAGE PAIRS:"; -while(<CONF>) { - chomp; - next if /^#/; - next if /^\s*$/; - s/^\s+//; - s/\s+$//; - my ($name, $path, $corpus, $lm, $xgrammar, $dev, $devref, @xtests) = split /\s+/; - $paths{$name} = $path; - $corpora{$name} = $corpus; - $lms{$name} = $lm; - $xgrammars{$name} = $xgrammar; - $devs{$name} = $dev; - $devrefs{$name} = $devref; - $tests{$name} = $xtests[0]; - $testevals{$name} = $xtests[1]; - print STDERR " $name"; -} -print STDERR "\n"; - -my %langpairs = map { $_ => 1 } qw( btec zhen fbis aren uren nlfr ); - -my $outdir = "$CWD/exp"; -my $help; -my $XFEATS; -my $EXTRA_FILTER = ''; -my $dataDir = '/export/ws10smt/data'; -if (GetOptions( - "data=s" => \$dataDir, - "xfeats" => \$XFEATS, -) == 0 || @ARGV!=2 || $help) { - print_help(); - exit; -} -my $lp = $ARGV[0]; -my $grammar = $ARGV[1]; -print STDERR " CORPUS REPO: $dataDir\n"; -print STDERR " LANGUAGE PAIR: $lp\n"; -die "I don't know about that language pair\n" unless $paths{$lp}; -my $corpdir = "$dataDir"; -if ($paths{$lp} =~ /^\//) { $corpdir = $paths{$lp}; } else { $corpdir .= '/' . $paths{$lp}; } -die "I can't find the corpora directory: $corpdir" unless -d $corpdir; -print STDERR " GRAMMAR: $grammar\n"; -my $LANG_MODEL = mydircat($corpdir, $lms{$lp}); -print STDERR " LM: $LANG_MODEL\n"; -my $CORPUS = mydircat($corpdir, $corpora{$lp}); -die "Can't find corpus: $CORPUS" unless -f $CORPUS; - -my $dev = mydircat($corpdir, $devs{$lp}); -my $drefs = $devrefs{$lp}; -die "Can't find dev: $dev\n" unless -f $dev; -die "Dev refs not set" unless $drefs; -$drefs = mydircat($corpdir, $drefs); - -my $test = mydircat($corpdir, $tests{$lp}); -my $teval = mydircat($corpdir, $testevals{$lp}); -die "Can't find test: $test\n" unless -f $test; -assert_exec($teval); - -if ($XFEATS) { - my $xgram = mydircat($corpdir, $xgrammars{$lp}); - die "Can't find x-grammar: $xgram" unless -f $xgram; - $EXTRA_FILTER = "$ADDXFEATS $xgram |"; - print STDERR "ADDING X-FEATS FROM $xgram\n"; -} - -# MAKE DEV -print STDERR "\nFILTERING FOR dev...\n"; -print STDERR "DEV: $dev (REFS=$drefs)\n"; -`mkdir -p $outdir`; -my $devgrammar = filter($grammar, $dev, 'dev', $outdir); -my $devini = mydircat($outdir, "cdec-dev.ini"); -write_cdec_ini($devini, $devgrammar); - - -# MAKE TEST -print STDERR "\nFILTERING FOR test...\n"; -print STDERR "TEST: $test (EVAL=$teval)\n"; -`mkdir -p $outdir`; -my $testgrammar = filter($grammar, $test, 'test', $outdir); -my $testini = mydircat($outdir, "cdec-test.ini"); -write_cdec_ini($testini, $testgrammar); - - -# CREATE INIT WEIGHTS -print STDERR "\nCREATING INITIAL WEIGHTS FILE: weights.init\n"; -my $weights = mydircat($outdir, "weights.init"); -write_random_weights_file($weights); - - -# VEST -print STDERR "\nMINIMUM ERROR TRAINING\n"; -my $tuned_weights = mydircat($outdir, 'weights.tuned'); -if (-f $tuned_weights) { - print STDERR "TUNED WEIGHTS $tuned_weights EXISTS: REUSING\n"; -} else { - my $cmd = "$DISTVEST --ref-files=$drefs --source-file=$dev --weights $weights $devini"; - print STDERR "MERT COMMAND: $cmd\n"; - `rm -rf $outdir/vest 2> /dev/null`; - chdir $outdir or die "Can't chdir to $outdir: $!"; - $weights = `$cmd`; - die "MERT reported non-zero exit code" unless $? == 0; - chomp $weights; - safesystem($tuned_weights, "cp $weights $tuned_weights"); - print STDERR "TUNED WEIGHTS: $tuned_weights\n"; - die "$tuned_weights is missing!" unless -f $tuned_weights; -} - -# DECODE -print STDERR "\nDECODE TEST SET\n"; -my $decolog = mydircat($outdir, "test-decode.log"); -my $testtrans = mydircat($outdir, "test.trans"); -my $cmd = "cat $test | $PARALLELIZE -j 20 -e $decolog -- $CDEC -c $testini -w $tuned_weights > $testtrans"; -safesystem($testtrans, $cmd) or die "Failed to decode test set!"; - - -# EVALUATE -print STDERR "\nEVALUATE TEST SET\n"; -print STDERR "TEST: $testtrans\n"; -$cmd = "$teval $testtrans"; -safesystem(undef, $cmd) or die "Failed to evaluate!"; -exit 0; - - -sub write_random_weights_file { - my ($file, @extras) = @_; - open F, ">$file" or die "Can't write $file: $!"; - my @feats = (@DEFAULT_FEATS, @extras); - if ($XFEATS) { - my @xfeats = qw( - X_LogRuleCount X_LogECount X_LogFCount X_EGivenF X_FGivenE X_SingletonRule X_SingletonE X_SingletonF - ); - @feats = (@feats, @xfeats); - } - for my $feat (@feats) { - my $r = rand(1.6); - my $w = $init_weights{$feat} * $r; - if ($w == 0) { $w = 0.0001; print STDERR "WARNING: $feat had no initial weight!\n"; } - print F "$feat $w\n"; - } - close F; -} - -sub filter { - my ($grammar, $set, $name, $outdir) = @_; - my $outgrammar = mydircat($outdir, "$name.scfg.gz"); - if (-f $outgrammar) { print STDERR "$outgrammar exists - REUSING!\n"; } else { - my $cmd = "gunzip -c $grammar | $FILTSCORE -c $CORPUS -t $set | $EXTRA_FILTER gzip > $outgrammar"; - safesystem($outgrammar, $cmd) or die "Can't filter and score grammar!"; - } - return $outgrammar; -} - -sub mydircat { - my ($base, $suffix) = @_; - if ($suffix =~ /^\//) { return $suffix; } - my $res = $base . '/' . $suffix; - $res =~ s/\/\//\//g; - return $res; -} - -sub write_cdec_ini { - my ($filename, $grammar_path) = (@_); - open CDECINI, ">$filename" or die "Can't write $filename: $!"; - print CDECINI <<EOT; -formalism=scfg -cubepruning_pop_limit=100 -add_pass_through_rules=true -scfg_extra_glue_grammar=/export/ws10smt/data/glue/glue.scfg.gz -grammar=$grammar_path -feature_function=WordPenalty -feature_function=LanguageModel -o 3 $LANG_MODEL -EOT - close CDECINI; -}; - -sub print_help { - print STDERR<<EOT; - -Usage: $0 [OPTIONS] language-pair unfiltered-grammar.gz - -Given an induced grammar for an entire corpus (i.e., generated by -local-gi-pipeline.pl), filter and featurize it for a dev and test set, -run MERT, report scores. - -EOT -} - -sub safesystem { - my $output = shift @_; - print STDERR "Executing: @_\n"; - system(@_); - if ($? == -1) { - print STDERR "ERROR: Failed to execute: @_\n $!\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - exit(1); - } - elsif ($? & 127) { - printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - exit(1); - } - else { - my $exitcode = $? >> 8; - if ($exitcode) { - print STDERR "Exit code: $exitcode\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - } - return ! $exitcode; - } -} - -sub assert_exec { - my @files = @_; - for my $file (@files) { - die "Can't find $file - did you run make?\n" unless -e $file; - die "Can't execute $file" unless -e $file; - } -}; - diff --git a/gi/pipeline/backoff-pipe.pl b/gi/pipeline/backoff-pipe.pl deleted file mode 100644 index ac103c8b..00000000 --- a/gi/pipeline/backoff-pipe.pl +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -use Getopt::Long "GetOptions"; - -my @grammars; -my $OUTPUTPREFIX = './giwork/bo.hier.grammar'; -safemkdir($OUTPUTPREFIX); -my $backoff_levels = 1; -my $glue_levels = 1; - -usage() unless &GetOptions('grmr=s@' => \ @grammars, - 'outprefix=s' => \ $OUTPUTPREFIX, - 'bo-lvls=i' => \ $backoff_levels, - 'glue-lvls=i' => \ $glue_levels, -); - -my $OUTDIR = $OUTPUTPREFIX . '/hier'; -print STDERR "@grammars\n"; - - -my %grmr = (); -foreach my $grammar (@grammars) { - $grammar =~ m/\/[^\/]*\.t(\d+)\.[^\/]*/; - $grmr{$1} = $grammar; -} - -my @index = sort keys %grmr; -$OUTDIR = $OUTDIR . join('-',@index); -safemkdir($OUTDIR); -my $BACKOFF_GRMR = $OUTDIR . '/backoff.hier.gz'; -safesystem("echo \"\" | gzip > $BACKOFF_GRMR"); -my $GLUE_GRMR = $OUTDIR . '/glue.hier.gz'; -safesystem("echo \"\" | gzip > $GLUE_GRMR"); -my $joinedgrammars = $OUTDIR . '/grammar.hier.gz'; - -join_grammars(); - -for my $i (0..(scalar @index)-2) { - my $freqs = extract_freqs($index[$i], $index[$i+1]); - if ($i < $backoff_levels) { - create_backoff_rules($index[$i],$index[$i+1],$freqs); - } - if ($i < $glue_levels) { - add_glue_rules($index[$i]); - } -} - -output_grammar_info(); - - -sub usage { - print <<EOT; - -Usage: $0 [OPTIONS] corpus.fr-en-al - -Induces a grammar using Pitman-Yor topic modeling or Posterior Regularisation. - -EOT - exit 1; -}; - -sub safemkdir { - my $dir = shift; - if (-d $dir) { return 1; } - return mkdir($dir); -} - - -sub safesystem { - print STDERR "Executing: @_\n"; - system(@_); - if ($? == -1) { - print STDERR "ERROR: Failed to execute: @_\n $!\n"; - exit(1); - } - elsif ($? & 127) { - printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - exit(1); - } - else { - my $exitcode = $? >> 8; - print STDERR "Exit code: $exitcode\n" if $exitcode; - return ! $exitcode; - } -} - - -sub join_grammars { - print STDERR "\n!!! JOINING GRAMMARS\n"; - if(-e $joinedgrammars) { - print STDERR "$joinedgrammars exists, reusing...\n"; - return; - } - safesystem("echo \"\" | gzip > $joinedgrammars"); - foreach my $i (@index) { - my $g = $grmr{$i}; - safesystem("zcat $g | sed -r -e 's/X([0-9]+)/X$i\\1/g' - | gzip > $g.2.gz"); - safesystem("zcat $joinedgrammars $g.2.gz | gzip > $joinedgrammars.2.gz"); - safesystem("mv $joinedgrammars.2.gz $joinedgrammars"); - } -} - - -sub extract_freqs { - my($grmr1,$grmr2) = @_; - print STDERR "\n!!!EXTRACTING FREQUENCIES: $grmr1->$grmr2\n"; - my $IN_COARSE = substr($grmr{$grmr1},0,index($grmr{$grmr1},".grammar/")) . "/labeled_spans.txt"; - my $IN_FINE = substr($grmr{$grmr2},0,index($grmr{$grmr2},".grammar/")) . "/labeled_spans.txt"; - my $OUT_SPANS = "$OUTDIR/labeled_spans.hier$grmr1-$grmr2.txt"; - my $FREQS = "$OUTDIR/label_freq.hier$grmr1-$grmr2.txt"; - if(-e $OUT_SPANS && -e $FREQS) { - print STDERR "$OUT_SPANS exists, reusing...\n"; - print STDERR "$FREQS exists, reusing...\n"; - return $FREQS; - } - - safesystem("paste -d ' ' $IN_COARSE $IN_FINE > $OUT_SPANS"); - - my %FREQ_HIER = (); - my %finehier = (); - - open SPANS, $OUT_SPANS or die $!; - while (<SPANS>) { - my ($tmp, $coarse, $fine) = split /\|\|\|/; - my @coarse_spans = $coarse =~ /\d+-\d+:X(\d+)/g; - my @fine_spans = $fine =~ /\d+-\d+:X(\d+)/g; - - foreach my $i (0..(scalar @coarse_spans)-1) { - my $coarse_cat = $coarse_spans[$i]; - my $fine_cat = $fine_spans[$i]; - - $FREQ_HIER{$coarse_cat}{$fine_cat}++; - } - } - close SPANS; - foreach (values %FREQ_HIER) { - my $coarse_freq = $_; - my $total = 0; - $total+=$_ for (values %{ $coarse_freq }); - $coarse_freq->{$_}=log($coarse_freq->{$_}/$total) for (keys %{ $coarse_freq }); - } - open FREQS, ">", $FREQS or die $!; - foreach my $coarse_cat (keys %FREQ_HIER) { - print FREQS "$coarse_cat |||"; - foreach my $fine_cat (keys %{$FREQ_HIER{$coarse_cat}}) { - my $freq = $FREQ_HIER{$coarse_cat}{$fine_cat}; - print FREQS " $fine_cat:$freq"; - if(! exists $finehier{$fine_cat} || $finehier{$fine_cat} < $freq) { - $finehier{$fine_cat} = $coarse_cat; - } - } - print FREQS "\n"; - } -# foreach my $fine_cat (keys %finehier) { -# print FREQS "$fine_cat -> $finehier{$fine_cat}\n"; -# } - close FREQS; - return $FREQS; -} - - -sub create_backoff_rules { - print STDERR "\n!!! CREATING BACKOFF RULES\n"; - my ($grmr1, $grmr2, $freq) = @_; - my $OUTFILE = "$OUTDIR/backoff.hier$grmr1-$grmr2.txt"; - if(-e $OUTFILE) { - print STDERR "$OUTFILE exists, reusing...\n"; - return; - } - open FREQS, $freq or die $!; - open TMP, ">", $OUTFILE or die $!; - while (<FREQS>) { - my $line = $_; - $line = m/^(\d+) \|\|\| (.+)$/; - my $coarse = $1; - $line = $2; - my @finefreq = $line =~ m/(\d+):(\S+)/g; - for(my $i = 0; $i < scalar @finefreq; $i+=2) { - my $finecat = $finefreq[$i]; - my $finefreq = $finefreq[$i+1]; - print TMP "[X$grmr1$coarse] ||| [X$grmr2$finecat,1]\t[1] ||| BackoffRule=$finefreq A=0-0\n"; - } - } - close TMP; - close FREQS; - safesystem("zcat $BACKOFF_GRMR | cat - $OUTFILE | gzip > $BACKOFF_GRMR.2.gz"); - safesystem("mv $BACKOFF_GRMR.2.gz $BACKOFF_GRMR"); -} - -sub add_glue_rules { - print STDERR "\n!!! CREATING GLUE RULES\n"; - my ($grmr) = @_; - my $OUTFILE = "$OUTDIR/glue.$grmr.gz"; - if (-e $OUTFILE) { - print STDERR "$OUTFILE exists, reusing...\n"; - return; - } - open TMP, ">", $OUTFILE or die $!; - for my $i (0..($grmr-1)) { - print TMP "[S] ||| [S,1] [X$grmr$i,2] ||| [1] [2] ||| Glue=1\n"; - print TMP "[S] ||| [X$grmr$i,1] ||| [1] ||| GlueTop=1\n"; - } - close TMP; - safesystem("zcat $GLUE_GRMR | cat - $OUTFILE | gzip > $GLUE_GRMR.2.gz"); - safesystem("mv $GLUE_GRMR.2.gz $GLUE_GRMR"); -} - -sub output_grammar_info { - print STDERR "\n!!! GRAMMAR INFORMATION\n"; - print STDOUT "GRAMMAR: \t$joinedgrammars\n"; - print STDOUT "GLUE: \t$GLUE_GRMR\n"; - print STDOUT "BACKOFF: \t$BACKOFF_GRMR\n"; -} diff --git a/gi/pipeline/blacklight.config b/gi/pipeline/blacklight.config deleted file mode 100644 index fc59a604..00000000 --- a/gi/pipeline/blacklight.config +++ /dev/null @@ -1,9 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM dev dev-refs test1 testt-eval.sh ... -/usr/users/0/cdyer/ws10smt/data -btec /home/cdyer/ws10smt-data/btec/ split.zh-en.al lm/en.3gram.lm.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -zhen /home/cdyer/ws10smt-data/chinese-english corpus.zh-en.al lm/c2e.3gram.lm.gz dev_and_test/mt02.src.txt dev_and_test/mt02.ref.* dev_and_test/mt03.src.txt eval-mt03.sh -aren /home/cdyer/ws10smt-data/arabic-english corpus.ar-en-al lm/a2e.3gram.lm.gz dev_and_test/dev.src.txt dev_and_test/dev.ref.txt.* dev_and_test/mt05.src.txt eval-mt05.sh -uren /usr/users/0/cdyer/ws10smt/data/urdu-english corpus.ur-en.al lm/u2e.en.lm.gz dev/dev.ur dev/dev.en* devtest/devtest.ur eval-devtest.sh -nlfr /home/cdyer/ws10smt-data/dutch-french corpus.nl-fr.al - diff --git a/gi/pipeline/clsp.config b/gi/pipeline/clsp.config deleted file mode 100644 index c23d409f..00000000 --- a/gi/pipeline/clsp.config +++ /dev/null @@ -1,10 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM dev dev-refs test1 testt-eval.sh ... -/export/ws10smt/data -btec /export/ws10smt/data/btec/ split.zh-en.al lm/en.3gram.lm.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -fbis /export/ws10smt/data/chinese-english.fbis corpus.zh-en.al -zhen /export/ws10smt/data/chinese-english corpus.zh-en.al lm/c2e.3gram.lm.gz dev_and_test/mt02.src.txt dev_and_test/mt02.ref.* dev_and_test/mt03.src.txt eval-mt03.sh -aren /export/ws10smt/data/arabic-english corpus.ar-en-al lm/a2e.3gram.lm.gz dev_and_test/dev.src.txt dev_and_test/dev.ref.txt.* dev_and_test/mt05.src.txt eval-mt05.sh -uren /export/ws10smt/data/urdu-english corpus.ur-en.al lm/u2e.en.lm.gz dev/dev.ur dev/dev.en* devtest/devtest.ur eval-devtest.sh -nlfr /export/ws10smt/data/dutch-french corpus.nl-fr.al - diff --git a/gi/pipeline/evaluation-pipeline.pl b/gi/pipeline/evaluation-pipeline.pl deleted file mode 100755 index 4b4529d9..00000000 --- a/gi/pipeline/evaluation-pipeline.pl +++ /dev/null @@ -1,364 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use Getopt::Long; -use Cwd; -my $CWD = getcwd; - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR, "$SCRIPT_DIR/../../environment"; } -use LocalConfig; - -my $JOBS = 15; -my $PMEM = "9G"; -my $NUM_TRANSLATIONS = 50; -my $GOAL = "S"; - -# featurize_grammar may add multiple features from a single feature extractor -# the key in this map is the extractor name, the value is a list of the extracted features -my $feat_map = { - "LogRuleCount" => [ "LogRuleCount", "SingletonRule" ] , -# "XFeatures" => [ "XFE","XEF" ] , - "XFeatures" => [ "XFE","XEF","LabelledEF","LabelledFE"], # ,"XE_Singleton","XF_Singleton"] , - "LabelledRuleConditionals" => [ "LabelledFE","LabelledEF" ] , - "LexProb" => [ "LexE2F", "LexF2E" ] , - "BackoffRule" => [ "BackoffRule" ] , - "RulePenalty" => [ "RulePenalty" ] , - "LHSProb" => [ "LHSProb" ] , - "LabellingShape" => [ "LabellingShape" ] , - "GenerativeProb" => [ "GenerativeProb" ] , -}; - -my %init_weights = qw( - EGivenF -0.735245 - FGivenE -0.219391 - Glue -0.306709 - GlueTop 0.0473331 - LanguageModel 2.40403 - LexE2F -0.266989 - LexF2E -0.550373 - LogECount -0.129853 - LogFCount -0.194037 - LogRuleCount 0.256706 - BackoffRule 0.5 - XFE -0.256706 - XEF -0.256706 - XF_Singleton -0.05 - XE_Singleton -0.8 - LabelledFE -0.256706 - LabelledEF -0.256706 - PassThrough -0.9304905 - SingletonE -3.04161 - SingletonF 0.0714027 - SingletonRule -0.889377 - WordPenalty -1.99495 - RulePenalty -0.1 - LabellingShape -0.1 - LHSProb -0.1 - GenerativeProb -0.1 -); - - -# these features are included by default -my @DEFAULT_FEATS = qw( PassThrough Glue GlueTop LanguageModel WordPenalty ); - - - -my $FILTERBYF = "$SCRIPT_DIR/scripts/filter-by-f.pl"; -my $CDEC = "$SCRIPT_DIR/../../decoder/cdec"; -my $PARALLELIZE = "$SCRIPT_DIR/../../vest/parallelize.pl"; -my $EXTOOLS = "$SCRIPT_DIR/../../extools"; -die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS; -my $VEST = "$SCRIPT_DIR/../../vest"; -die "Can't find vest: $VEST" unless -e $VEST && -d $VEST; -my $DISTVEST = "$VEST/dist-vest.pl"; -my $FILTER = "$EXTOOLS/filter_grammar"; -my $FEATURIZE = "$EXTOOLS/featurize_grammar"; -assert_exec($CDEC, $PARALLELIZE, $FILTER, $FEATURIZE, $DISTVEST, $FILTERBYF); - -my $numtopics = 25; - -my $config = "$SCRIPT_DIR/" . (lc environment_name()) . '.config'; -print STDERR "CORPORA CONFIGURATION: $config\n"; -open CONF, "<$config" or die "Can't read $config: $!"; -my %paths; -my %corpora; -my %lms; -my %devs; -my %devrefs; -my %tests; -my %testevals; -my $datadir; -print STDERR " LANGUAGE PAIRS:"; -while(<CONF>) { - chomp; - next if /^#/; - next if /^\s*$/; - s/^\s+//; - s/\s+$//; - if (! defined $datadir) { $datadir = $_; next; } - my ($name, $path, $corpus, $lm, $dev, $devref, @xtests) = split /\s+/; - $paths{$name} = $path; - $corpora{$name} = $corpus; - $lms{$name} = $lm; - $devs{$name} = $dev; - $devrefs{$name} = $devref; - $tests{$name} = $xtests[0]; - $testevals{$name} = $xtests[1]; - print STDERR " $name"; -} -print STDERR "\n"; - -my %langpairs = map { $_ => 1 } qw( btec zhen fbis aren uren nlfr ); - -my $outdir = "$CWD/exp"; -my $help; -my $FEATURIZER_OPTS = ''; -my $dataDir = '/export/ws10smt/data'; -my @features; -my $bkoffgram; -my $gluegram; -my $oovgram; -my $usefork; -my $lmorder = 3; -my $density; -if (GetOptions( - "backoff-grammar=s" => \$bkoffgram, - "density-prune=f" => \$density, - "glue-grammar=s" => \$gluegram, - "oov-grammar=s" => \$oovgram, - "data=s" => \$dataDir, - "pmem=s" => \$PMEM, - "n=i" => \$NUM_TRANSLATIONS, - "features=s@" => \@features, - "use-fork" => \$usefork, - "jobs=i" => \$JOBS, - "out-dir=s" => \$outdir, - "lmorder=i" => \$lmorder, - "goal=s" => \$GOAL, -) == 0 || @ARGV!=2 || $help) { - print_help(); - exit; -} -my $DENSITY_PRUNE = ''; -if ($density) { - $DENSITY_PRUNE = "--density-prune $density"; -} -if ($usefork) { $usefork="--use-fork"; } else { $usefork = ''; } -my @fkeys = keys %$feat_map; -die "You must specify one or more features with -f. Known features: @fkeys\n" unless scalar @features > 0; -my @xfeats; -for my $feat (@features) { - my $rs = $feat_map->{$feat}; - if (!defined $rs) { die "DON'T KNOW ABOUT FEATURE $feat\n"; } - my @xfs = @$rs; - @xfeats = (@xfeats, @xfs); - $FEATURIZER_OPTS .= " -f $feat" unless $feat eq "BackoffRule"; -} -print STDERR "X-FEATS: @xfeats\n"; - -my $lp = $ARGV[0]; -my $grammar = $ARGV[1]; -print STDERR " CORPUS REPO: $dataDir\n"; -print STDERR " LANGUAGE PAIR: $lp\n"; -die "I don't know about that language pair\n" unless $paths{$lp}; -my $corpdir = "$dataDir"; -if ($paths{$lp} =~ /^\//) { $corpdir = $paths{$lp}; } else { $corpdir .= '/' . $paths{$lp}; } -die "I can't find the corpora directory: $corpdir" unless -d $corpdir; -print STDERR " GRAMMAR: $grammar\n"; -my $LANG_MODEL = mydircat($corpdir, $lms{$lp}); -print STDERR " LM: $LANG_MODEL\n"; -my $CORPUS = mydircat($corpdir, $corpora{$lp}); -die "Can't find corpus: $CORPUS" unless -f $CORPUS; - -my $dev = mydircat($corpdir, $devs{$lp}); -my $drefs = $devrefs{$lp}; -die "Can't find dev: $dev\n" unless -f $dev; -die "Dev refs not set" unless $drefs; -$drefs = mydircat($corpdir, $drefs); - -my $test = mydircat($corpdir, $tests{$lp}); -my $teval = mydircat($corpdir, $testevals{$lp}); -#die "Can't find test: $test\n" unless -f $test; -#assert_exec($teval); - -`mkdir -p $outdir`; - -# CREATE INIT WEIGHTS -print STDERR "\nCREATING INITIAL WEIGHTS FILE: weights.init\n"; -my $weights = mydircat($outdir, "weights.init"); -write_random_weights_file($weights, @xfeats); - -my $bkoff_grmr; -my $glue_grmr; -if($bkoffgram) { - print STDERR "Placing backoff grammar…\n"; - $bkoff_grmr = mydircat($outdir, "backoff.scfg.gz"); - print STDERR "cp $bkoffgram $bkoff_grmr\n"; - safesystem(undef,"cp $bkoffgram $bkoff_grmr"); -} -if($gluegram) { - print STDERR "Placing glue grammar…\n"; - $glue_grmr = mydircat($outdir, "glue.bo.scfg.gz"); - print STDERR "cp $gluegram $glue_grmr\n"; - safesystem(undef,"cp $gluegram $glue_grmr"); -} - -# MAKE DEV -print STDERR "\nFILTERING FOR dev...\n"; -print STDERR "DEV: $dev (REFS=$drefs)\n"; -my $devgrammar = filter($grammar, $dev, 'dev', $outdir); -my $devini = mydircat($outdir, "cdec-dev.ini"); -write_cdec_ini($devini, $devgrammar); - - -# MAKE TEST -print STDERR "\nFILTERING FOR test...\n"; -print STDERR "TEST: $test (EVAL=$teval)\n"; -`mkdir -p $outdir`; -my $testgrammar = filter($grammar, $test, 'test', $outdir); -my $testini = mydircat($outdir, "cdec-test.ini"); -write_cdec_ini($testini, $testgrammar); - - -# VEST -print STDERR "\nMINIMUM ERROR TRAINING\n"; -my $tuned_weights = mydircat($outdir, 'weights.tuned'); -if (-f $tuned_weights) { - print STDERR "TUNED WEIGHTS $tuned_weights EXISTS: REUSING\n"; -} else { - my $cmd = "$DISTVEST $usefork $DENSITY_PRUNE --decode-nodes $JOBS --pmem=$PMEM --ref-files=$drefs --source-file=$dev --weights $weights $devini"; - print STDERR "MERT COMMAND: $cmd\n"; - `rm -rf $outdir/vest 2> /dev/null`; - chdir $outdir or die "Can't chdir to $outdir: $!"; - $weights = `$cmd`; - die "MERT reported non-zero exit code" unless $? == 0; - chomp $weights; - safesystem($tuned_weights, "cp $weights $tuned_weights"); - print STDERR "TUNED WEIGHTS: $tuned_weights\n"; - die "$tuned_weights is missing!" unless -f $tuned_weights; -} - -# DECODE -print STDERR "\nDECODE TEST SET\n"; -my $decolog = mydircat($outdir, "test-decode.log"); -my $testtrans = mydircat($outdir, "test.trans"); -my $cmd = "cat $test | $PARALLELIZE $usefork -j $JOBS -e $decolog -- $CDEC -c $testini -w $tuned_weights > $testtrans"; -safesystem($testtrans, $cmd) or die "Failed to decode test set!"; - - -# EVALUATE -print STDERR "\nEVALUATE TEST SET\n"; -print STDERR "TEST: $testtrans\n"; -$cmd = "$teval $testtrans"; -safesystem(undef, $cmd) or die "Failed to evaluate!"; -exit 0; - - -sub write_random_weights_file { - my ($file, @extras) = @_; - if (-f $file) { - print STDERR "$file exists - REUSING!\n"; - return; - } - open F, ">$file" or die "Can't write $file: $!"; - my @feats = (@DEFAULT_FEATS, @extras); - for my $feat (@feats) { - my $r = rand(0.4) + 0.8; - my $w = $init_weights{$feat} * $r; - if ($w == 0) { $w = 0.0001; print STDERR "WARNING: $feat had no initial weight!\n"; } - print F "$feat $w\n"; - } - close F; -} - -sub filter { - my ($grammar, $set, $name, $outdir) = @_; - my $out1 = mydircat($outdir, "$name.filt.gz"); - my $out2 = mydircat($outdir, "$name.f_feat.gz"); - my $outgrammar = mydircat($outdir, "$name.scfg.gz"); - if (-f $outgrammar) { print STDERR "$outgrammar exists - REUSING!\n"; } else { - my $cmd = "gunzip -c $grammar | $FILTER -t $set | gzip > $out1"; - safesystem($out1, $cmd) or die "Filtering failed."; - $cmd = "gunzip -c $out1 | $FEATURIZE $FEATURIZER_OPTS -g $out1 -c $CORPUS | gzip > $out2"; - safesystem($out2, $cmd) or die "Featurizing failed"; - $cmd = "$FILTERBYF $NUM_TRANSLATIONS $out2 $outgrammar"; - safesystem($outgrammar, $cmd) or die "Secondary filtering failed"; - } - return $outgrammar; -} - -sub mydircat { - my ($base, $suffix) = @_; - if ($suffix =~ /^\//) { return $suffix; } - my $res = $base . '/' . $suffix; - $res =~ s/\/\//\//g; - return $res; -} - -sub write_cdec_ini { - my ($filename, $grammar_path) = (@_); - open CDECINI, ">$filename" or die "Can't write $filename: $!"; - my $glue = ($gluegram ? "$glue_grmr" : "$datadir/glue/glue.scfg.gz"); - my $oov = ($oovgram ? "$oovgram" : "$datadir/oov.scfg.gz"); - print CDECINI <<EOT; -formalism=scfg -cubepruning_pop_limit=100 -add_pass_through_rules=true -scfg_extra_glue_grammar=$glue -grammar=$oov -grammar=$grammar_path -scfg_default_nt=OOV -scfg_no_hiero_glue_grammar=true -feature_function=WordPenalty -feature_function=LanguageModel -o $lmorder $LANG_MODEL -goal=$GOAL -EOT - print CDECINI "grammar=$bkoff_grmr\n" if $bkoffgram; - close CDECINI; -}; - -sub print_help { - print STDERR<<EOT; - -Usage: $0 [-c data-config-file] [-n N] language-pair grammar.bidir.gz [OPTIONS] - -Given an induced grammar for an entire corpus (i.e., generated by -local-gi-pipeline.pl), filter and featurize it for a dev and test set, -run MERT, report scores. Use -n to specify the number of translations -to keep for a given source (30 is default). - -EOT -} - -sub safesystem { - my $output = shift @_; - print STDERR "Executing: @_\n"; - system(@_); - if ($? == -1) { - print STDERR "ERROR: Failed to execute: @_\n $!\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - exit(1); - } - elsif ($? & 127) { - printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - exit(1); - } - else { - my $exitcode = $? >> 8; - if ($exitcode) { - print STDERR "Exit code: $exitcode\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - } - return ! $exitcode; - } -} - -sub assert_exec { - my @files = @_; - for my $file (@files) { - die "Can't find $file - did you run make?\n" unless -e $file; - die "Can't execute $file" unless -e $file; - } -}; - diff --git a/gi/pipeline/local-gi-pipeline.pl b/gi/pipeline/local-gi-pipeline.pl deleted file mode 100755 index e31167a2..00000000 --- a/gi/pipeline/local-gi-pipeline.pl +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use File::Copy; - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path cwd /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } - -use Getopt::Long "GetOptions"; - -my $GZIP = 'gzip'; -my $ZCAT = 'gunzip -c'; -my $SED = 'sed -e'; -my $BASE_PHRASE_MAX_SIZE = 10; -my $COMPLETE_CACHE = 1; -my $ITEMS_IN_MEMORY = 10000000; # cache size in extractors -my $NUM_TOPICS = 50; -my $NUM_TOPICS_COARSE; -my $NUM_TOPICS_FINE = $NUM_TOPICS; -my $NUM_SAMPLES = 1000; -my $CONTEXT_SIZE = 1; -my $BIDIR = 0; -my $TOPICS_CONFIG = "pyp-topics.conf"; -my $LANGUAGE = "target"; -my $LABEL_THRESHOLD = "0"; -my $PRESERVE_PHRASES; - -my $MODEL = "pyp"; -my $NUM_ITERS = 100; -my $PR_SCALE_P = 0; -my $PR_SCALE_C = 0; -my $PR_FLAGS = ""; -my $MORFMARK = ""; - -my $EXTOOLS = "$SCRIPT_DIR/../../extools"; -die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS; -my $PYPTOOLS = "$SCRIPT_DIR/../pyp-topics/src"; -die "Can't find pyp-topics: $PYPTOOLS" unless -e $PYPTOOLS && -d $PYPTOOLS; -my $PYPSCRIPTS = "$SCRIPT_DIR/../pyp-topics/scripts"; -die "Can't find pyp-topics: $PYPSCRIPTS" unless -e $PYPSCRIPTS && -d $PYPSCRIPTS; -my $PRTOOLS = "$SCRIPT_DIR/../posterior-regularisation"; -die "Can't find posterior-regularisation: $PRTOOLS" unless -e $PRTOOLS && -d $PRTOOLS; -my $REDUCER = "$EXTOOLS/mr_stripe_rule_reduce"; -my $C2D = "$PYPSCRIPTS/contexts2documents.py"; -my $S2L = "$PYPSCRIPTS/spans2labels.py"; -my $SPLIT = "$SCRIPT_DIR/../posterior-regularisation/split-languages.py"; - -my $PREM_TRAIN="$PRTOOLS/prjava/train-PR-cluster.sh"; - -my $SORT_KEYS = "$SCRIPT_DIR/scripts/sort-by-key.sh"; -my $PATCH_CORPUS = "$SCRIPT_DIR/scripts/patch-corpus.pl"; -my $REMOVE_TAGS_CORPUS = "$SCRIPT_DIR/scripts/remove-tags-from-corpus.pl"; -my $REMOVE_TAGS_CONTEXT = "$SCRIPT_DIR/scripts/remove-tags-from-contexts.pl"; -my $EXTRACTOR = "$EXTOOLS/extractor"; -my $TOPIC_TRAIN = "$PYPTOOLS/pyp-contexts-train"; -my $MORF_DOC_FILTER = "$SCRIPT_DIR/../morf-segmentation/filter_docs.pl"; - -assert_exec($PATCH_CORPUS, $SORT_KEYS, $REDUCER, $EXTRACTOR, - $S2L, $C2D, $TOPIC_TRAIN, $SPLIT, $REMOVE_TAGS_CONTEXT, $REMOVE_TAGS_CORPUS, $MORF_DOC_FILTER); - -my $BACKOFF_GRAMMAR; -my $DEFAULT_CAT; -my $HIER_CAT; -my %FREQ_HIER = (); -my $TAGGED_CORPUS; - -my $NAME_SHORTCUT; - -my $OUTPUT = './giwork'; -usage() unless &GetOptions('base_phrase_max_size=i' => \$BASE_PHRASE_MAX_SIZE, - 'backoff_grammar' => \$BACKOFF_GRAMMAR, - 'output=s' => \$OUTPUT, - 'model=s' => \$MODEL, - 'topics=i' => \$NUM_TOPICS_FINE, - 'coarse_topics=i' => \$NUM_TOPICS_COARSE, - 'trg_context=i' => \$CONTEXT_SIZE, - 'samples=i' => \$NUM_SAMPLES, - 'label_threshold=f' => \$LABEL_THRESHOLD, - 'use_default_cat' => \$DEFAULT_CAT, - 'topics-config=s' => \$TOPICS_CONFIG, - 'iterations=i' => \$NUM_ITERS, - 'pr-scale-phrase=f' => \$PR_SCALE_P, - 'pr-scale-context=f' => \$PR_SCALE_C, - 'pr-flags=s' => \$PR_FLAGS, - 'tagged_corpus=s' => \$TAGGED_CORPUS, - 'language=s' => \$LANGUAGE, - 'get_name_only' => \$NAME_SHORTCUT, - 'preserve_phrases' => \$PRESERVE_PHRASES, - 'morf=s' => \$MORFMARK, - ); -if ($NAME_SHORTCUT) { - $NUM_TOPICS = $NUM_TOPICS_FINE; - print STDERR labeled_dir(); - exit 0; -} -usage() unless scalar @ARGV == 1; -my $CORPUS = $ARGV[0]; -open F, "<$CORPUS" or die "Can't read $CORPUS: $!"; close F; - -$NUM_TOPICS = $NUM_TOPICS_FINE; - -$HIER_CAT = ( $NUM_TOPICS_COARSE ? 1 : 0 ); - -print STDERR " Output: $OUTPUT\n"; -my $DATA_DIR = $OUTPUT . '/corpora'; -my $LEX_NAME = "corpus.f_e_a.$LANGUAGE.lex"; -my $CORPUS_LEX = $DATA_DIR . '/' . $LEX_NAME; # corpus used to extract rules -my $CORPUS_CLUSTER = $DATA_DIR . "/corpus.f_e_a.$LANGUAGE.cluster"; # corpus used for clustering (often identical) - -my $CONTEXT_DIR = $OUTPUT . '/' . context_dir(); -my $CLUSTER_DIR = $OUTPUT . '/' . cluster_dir(); -my $LABELED_DIR = $OUTPUT . '/' . labeled_dir(); -my $CLUSTER_DIR_C; -my $CLUSTER_DIR_F; -my $LABELED_DIR_C; -my $LABELED_DIR_F; -if($HIER_CAT) { - $CLUSTER_DIR_F = $CLUSTER_DIR; - $LABELED_DIR_F = $LABELED_DIR; - $NUM_TOPICS = $NUM_TOPICS_COARSE; - $CLUSTER_DIR_C = $OUTPUT . '/' . cluster_dir(); - $LABELED_DIR_C = $OUTPUT . '/' . labeled_dir(); - $NUM_TOPICS = $NUM_TOPICS_FINE; -} -my $GRAMMAR_DIR = $OUTPUT . '/' . grammar_dir(); -print STDERR " Context: $CONTEXT_DIR\n Cluster: $CLUSTER_DIR\n Labeled: $LABELED_DIR\n Grammar: $GRAMMAR_DIR\n"; -safemkdir($OUTPUT) or die "Couldn't create output directory $OUTPUT: $!"; -safemkdir($DATA_DIR) or die "Couldn't create output directory $DATA_DIR: $!"; -safemkdir($CONTEXT_DIR) or die "Couldn't create output directory $CONTEXT_DIR: $!"; -safemkdir($CLUSTER_DIR) or die "Couldn't create output directory $CLUSTER_DIR: $!"; -if($HIER_CAT) { - safemkdir($CLUSTER_DIR_C) or die "Couldn't create output directory $CLUSTER_DIR_C: $!"; - safemkdir($LABELED_DIR_C) or die "Couldn't create output directory $LABELED_DIR_C: $!"; -} -safemkdir($LABELED_DIR) or die "Couldn't create output directory $LABELED_DIR: $!"; -safemkdir($GRAMMAR_DIR) or die "Couldn't create output directory $GRAMMAR_DIR: $!"; -if(-e $TOPICS_CONFIG) { - copy($TOPICS_CONFIG, $CLUSTER_DIR) or die "Copy failed: $!"; -} - -setup_data(); - -if (lc($MODEL) eq "blagree") { - extract_bilingual_context(); -} else { - extract_context(); -} - -if (lc($MODEL) eq "pyp") { - if($HIER_CAT) { - $NUM_TOPICS = $NUM_TOPICS_COARSE; - $CLUSTER_DIR = $CLUSTER_DIR_C; - topic_train(); - $NUM_TOPICS = $NUM_TOPICS_FINE; - $CLUSTER_DIR = $CLUSTER_DIR_F; - topic_train(); - } else { - topic_train(); - } -} elsif (lc($MODEL) =~ /pr|em|agree/) { - prem_train(); -} else { die "Unsupported model type: $MODEL. Must be one of PYP or PREM.\n"; } -if($HIER_CAT) { - $NUM_TOPICS = $NUM_TOPICS_COARSE; - $CLUSTER_DIR = $CLUSTER_DIR_C; - $LABELED_DIR = $LABELED_DIR_C; - label_spans_with_topics(); - $NUM_TOPICS = $NUM_TOPICS_FINE; - $CLUSTER_DIR = $CLUSTER_DIR_F; - $LABELED_DIR = $LABELED_DIR_F; - label_spans_with_topics(); - extract_freqs(); -} else { - label_spans_with_topics(); -} -my $res; -if ($BIDIR) { - $res = grammar_extract_bidir(); -} else { - $res = grammar_extract(); -} -print STDERR "\n!!!COMPLETE!!!\n"; -print STDERR "GRAMMAR: $res\nYou should probably run: $SCRIPT_DIR/evaluation-pipeline.pl LANGPAIR giwork/ct1s0.L10.PYP.t4.s20.grammar/grammar.gz -f FEAT1 -f FEAT2\n\n"; -exit 0; - -sub setup_data { - print STDERR "\n!!!PREPARE CORPORA!!!\n"; - if (-f $CORPUS_LEX && $CORPUS_CLUSTER) { - print STDERR "$CORPUS_LEX and $CORPUS_CLUSTER exist, reusing...\n"; - return; - } - copy($CORPUS, $CORPUS_LEX); - if ($TAGGED_CORPUS) { - die "Can't find $TAGGED_CORPUS" unless -f $TAGGED_CORPUS; - my $opt=""; - $opt = "-s" if ($LANGUAGE eq "source"); - $opt = $opt . " -a" if ($PRESERVE_PHRASES); - my $cmd="$PATCH_CORPUS $opt $TAGGED_CORPUS $CORPUS_LEX > $CORPUS_CLUSTER"; - safesystem($cmd) or die "Failed to extract contexts."; - } else { - symlink($LEX_NAME, $CORPUS_CLUSTER); - } -} - -sub context_dir { - return "ct${CONTEXT_SIZE}s0.L$BASE_PHRASE_MAX_SIZE.l$LANGUAGE"; -} - -sub cluster_dir { - if (lc($MODEL) eq "pyp") { - return context_dir() . ".PYP.t$NUM_TOPICS.s$NUM_SAMPLES"; - } elsif (lc($MODEL) eq "em") { - return context_dir() . ".EM.t$NUM_TOPICS.i$NUM_ITERS"; - } elsif (lc($MODEL) eq "pr") { - return context_dir() . ".PR.t$NUM_TOPICS.i$NUM_ITERS.sp$PR_SCALE_P.sc$PR_SCALE_C"; - } elsif (lc($MODEL) eq "agree") { - return context_dir() . ".AGREE.t$NUM_TOPICS.i$NUM_ITERS"; - } elsif (lc($MODEL) eq "blagree") { - return context_dir() . ".BLAGREE.t$NUM_TOPICS.i$NUM_ITERS"; - } -} - -sub labeled_dir { - if (lc($MODEL) eq "pyp" && $LABEL_THRESHOLD ne "0") { - return cluster_dir() . "_lt$LABEL_THRESHOLD"; - } else { - return cluster_dir(); - } -} - -sub grammar_dir { - # TODO add grammar config options -- adjacent NTs, etc - if($HIER_CAT) { - return cluster_dir() . ".hier$NUM_TOPICS_COARSE-$NUM_TOPICS_FINE.grammar"; - } else { - return labeled_dir() . ".grammar"; - } -} - - - -sub safemkdir { - my $dir = shift; - if (-d $dir) { return 1; } - return mkdir($dir); -} - -sub usage { - print <<EOT; - -Usage: $0 [OPTIONS] corpus.fr-en-al - -Induces a grammar using Pitman-Yor topic modeling or Posterior Regularisation. - -EOT - exit 1; -}; - -sub assert_exec { - my @files = @_; - for my $file (@files) { - die "Can't find $file - did you run make?\n" unless -e $file; - die "Can't execute $file" unless -e $file; - } -}; - -sub extract_context { - print STDERR "\n!!!CONTEXT EXTRACTION\n"; - my $OUT_CONTEXTS = "$CONTEXT_DIR/context.txt.gz"; - if (-e $OUT_CONTEXTS) { - print STDERR "$OUT_CONTEXTS exists, reusing...\n"; - } else { - my $ccopt = "-c $ITEMS_IN_MEMORY"; - my $postsort = "| $REDUCER "; - if ($COMPLETE_CACHE) { - print STDERR "COMPLETE_CACHE is set: removing memory limits on cache.\n"; - $ccopt = "-c 0"; - $postsort = "" unless ($PRESERVE_PHRASES); - } - - my $presort = ($PRESERVE_PHRASES ? "| $REMOVE_TAGS_CONTEXT --phrase=tok --context=tag " : ""); - - if ($MORFMARK ne "") { - $presort = $presort . "| $MORF_DOC_FILTER \"$MORFMARK\" "; - } - - my $cmd = "$EXTRACTOR -i $CORPUS_CLUSTER $ccopt -L $BASE_PHRASE_MAX_SIZE -C -S $CONTEXT_SIZE --phrase_language $LANGUAGE --context_language $LANGUAGE $presort | $SORT_KEYS $postsort | $GZIP > $OUT_CONTEXTS"; - safesystem($cmd) or die "Failed to extract contexts."; - } -} - -sub extract_bilingual_context { - print STDERR "\n!!!CONTEXT EXTRACTION\n"; - my $OUT_SRC_CONTEXTS = "$CONTEXT_DIR/context.source"; - my $OUT_TGT_CONTEXTS = "$CONTEXT_DIR/context.target"; - - if (-e $OUT_SRC_CONTEXTS . ".gz" and -e $OUT_TGT_CONTEXTS . ".gz") { - print STDERR "$OUT_SRC_CONTEXTS.gz and $OUT_TGT_CONTEXTS.gz exist, reusing...\n"; - } else { - my $OUT_BI_CONTEXTS = "$CONTEXT_DIR/context.bilingual.txt.gz"; - my $cmd = "$EXTRACTOR -i $CORPUS_CLUSTER -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -C -S $CONTEXT_SIZE --phrase_language both --context_language both | $SORT_KEYS | $REDUCER | $GZIP > $OUT_BI_CONTEXTS"; - if ($COMPLETE_CACHE) { - print STDERR "COMPLETE_CACHE is set: removing memory limits on cache.\n"; - $cmd = "$EXTRACTOR -i $CORPUS_CLUSTER -c 0 -L $BASE_PHRASE_MAX_SIZE -C -S $CONTEXT_SIZE --phrase_language both --context_language both | $SORT_KEYS | $GZIP > $OUT_BI_CONTEXTS"; - } - safesystem($cmd) or die "Failed to extract contexts."; - - safesystem("$ZCAT $OUT_BI_CONTEXTS | $SPLIT $OUT_SRC_CONTEXTS $OUT_TGT_CONTEXTS") or die "Failed to split contexts.\n"; - safesystem("$GZIP -f $OUT_SRC_CONTEXTS") or die "Failed to zip output contexts.\n"; - safesystem("$GZIP -f $OUT_TGT_CONTEXTS") or die "Failed to zip output contexts.\n"; - } -} - - -sub topic_train { - print STDERR "\n!!!TRAIN PYP TOPICS\n"; - my $IN_CONTEXTS = "$CONTEXT_DIR/context.txt.gz"; - my $OUT_CLUSTERS = "$CLUSTER_DIR/docs.txt.gz"; - if (-e $OUT_CLUSTERS) { - print STDERR "$OUT_CLUSTERS exists, reusing...\n"; - } else { - safesystem("$TOPIC_TRAIN --data $IN_CONTEXTS --backoff-type simple -t $NUM_TOPICS -s $NUM_SAMPLES -o $OUT_CLUSTERS -c $TOPICS_CONFIG -w /dev/null") or die "Topic training failed.\n"; - } -} - -sub prem_train { - print STDERR "\n!!!TRAIN PR/EM model\n"; - my $OUT_CLUSTERS = "$CLUSTER_DIR/docs.txt.gz"; - if (-e $OUT_CLUSTERS) { - print STDERR "$OUT_CLUSTERS exists, reusing...\n"; - } else { - my $in = "--in $CONTEXT_DIR/context.txt.gz"; - my $opts = ""; - if (lc($MODEL) eq "pr") { - $opts = "--scale-phrase $PR_SCALE_P --scale-context $PR_SCALE_C"; - } elsif (lc($MODEL) eq "agree") { - $opts = "--agree-direction"; - } elsif (lc($MODEL) eq "blagree") { - $in = "--in $CONTEXT_DIR/context.source.gz --in1 $CONTEXT_DIR/context.target.gz"; - $opts = "--agree-language"; - } - safesystem("$PREM_TRAIN $in --topics $NUM_TOPICS --out $OUT_CLUSTERS --iterations $NUM_ITERS $opts $PR_FLAGS") or die "Topic training failed.\n"; - } -} - -sub label_spans_with_topics { - my ($file) = (@_); - print STDERR "\n!!!LABEL SPANS\n"; - my $IN_CLUSTERS = "$CLUSTER_DIR/docs.txt.gz"; - my $OUT_SPANS = "$LABELED_DIR/labeled_spans.txt"; - if (-e $OUT_SPANS) { - print STDERR "$OUT_SPANS exists, reusing...\n"; - } else { - my $extra = "tt"; - if ($LANGUAGE eq "source") { - $extra = "ss"; - } elsif ($LANGUAGE eq "both") { - $extra = "bb"; - } else { die "Invalid language specifier $LANGUAGE\n" unless $LANGUAGE eq "target" }; - $extra = $extra . " tok,tag" if ($PRESERVE_PHRASES); - safesystem("$ZCAT $IN_CLUSTERS > $CLUSTER_DIR/clusters.txt") or die "Failed to unzip"; - safesystem("$EXTRACTOR --base_phrase_spans -i $CORPUS_CLUSTER -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -S $CONTEXT_SIZE | $S2L $CLUSTER_DIR/clusters.txt $CONTEXT_SIZE $LABEL_THRESHOLD $extra > $OUT_SPANS") or die "Failed to label spans"; - unlink("$CLUSTER_DIR/clusters.txt") or warn "Failed to remove $CLUSTER_DIR/clusters.txt"; - safesystem("paste -d ' ' $CORPUS_LEX $OUT_SPANS | sed 's/ *||| *\$//' > $LABELED_DIR/corpus.src_trg_al_label") or die "Couldn't paste"; - } -} - -sub extract_freqs { - print STDERR "\n!!!EXTRACTING FREQUENCIES\n"; - my $IN_COARSE = "$LABELED_DIR_C/labeled_spans.txt"; - my $IN_FINE = "$LABELED_DIR_F/labeled_spans.txt"; - my $OUT_SPANS = "$LABELED_DIR_F/labeled_spans.hier$NUM_TOPICS_COARSE-$NUM_TOPICS_FINE.txt"; - my $FREQS = "$LABELED_DIR_F/label_freq.hier$NUM_TOPICS_COARSE-$NUM_TOPICS_FINE.txt"; - my $COARSE_EXPR = "\'s/\\(X[0-9][0-9]*\\)/\\1c/g\'"; #' - my $FINE_EXPR = "\'s/\\(X[0-9][0-9]*\\)/\\1f/g\'"; #' - my %finehier = (); - if (-e $OUT_SPANS) { - print STDERR "$OUT_SPANS exists, reusing...\n"; - } else { - safesystem("paste -d ' ' $IN_COARSE $IN_FINE > $OUT_SPANS"); - } - open SPANS, $OUT_SPANS or die $!; - while (<SPANS>) { - my ($tmp, $coarse, $fine) = split /\|\|\|/; - my @coarse_spans = $coarse =~ /\d+-\d+:X(\d+)/g; - my @fine_spans = $fine =~ /\d+-\d+:X(\d+)/g; - - foreach my $i (0..(scalar @coarse_spans)-1) { - my $coarse_cat = $coarse_spans[$i]; - my $fine_cat = $fine_spans[$i]; - - $FREQ_HIER{$coarse_cat}{$fine_cat}++; - } - } - close SPANS; - foreach (values %FREQ_HIER) { - my $coarse_freq = $_; - my $total = 0; - $total+=$_ for (values %{ $coarse_freq }); - $coarse_freq->{$_}=log($coarse_freq->{$_}/$total) for (keys %{ $coarse_freq }); - } - open FREQS, ">", $FREQS or die $!; - foreach my $coarse_cat (keys %FREQ_HIER) { - print FREQS "$coarse_cat |||"; - foreach my $fine_cat (keys %{$FREQ_HIER{$coarse_cat}}) { - my $res = $FREQ_HIER{$coarse_cat}{$fine_cat}; - print FREQS " $fine_cat:$res"; - if(! exists $finehier{$fine_cat} || $finehier{$fine_cat} < $res) { - $finehier{$fine_cat} = $coarse_cat; - } - } - print FREQS "\n"; - } -# foreach my $fine_cat (keys %finehier) { -# print FREQS "$fine_cat -> $finehier{$fine_cat}\n"; -# } - close FREQS; - $CLUSTER_DIR = $CLUSTER_DIR_F; -} - -sub grammar_extract { - my $LABELED = "$LABELED_DIR/corpus.src_trg_al_label"; - print STDERR "\n!!!EXTRACTING GRAMMAR\n"; - my $OUTGRAMMAR = "$GRAMMAR_DIR/grammar.gz"; - if (-e $OUTGRAMMAR) { - print STDERR "$OUTGRAMMAR exists, reusing...\n"; - } else { - my $BACKOFF_ARG = ($BACKOFF_GRAMMAR ? "-g" : ""); - my $DEFAULT_CAT_ARG = ($DEFAULT_CAT ? "-d X" : ""); - safesystem("$EXTRACTOR -i $LABELED -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -t $NUM_TOPICS $BACKOFF_ARG $DEFAULT_CAT_ARG | $SORT_KEYS | $REDUCER -p | $GZIP > $OUTGRAMMAR") or die "Couldn't extract grammar"; - } - return $OUTGRAMMAR; -} - -sub grammar_extract_bidir { -#gzcat ex.output.gz | ./mr_stripe_rule_reduce -p -b | sort -t $'\t' -k 1 | ./mr_stripe_rule_reduce | gzip > phrase-table.gz - my $LABELED = "$LABELED_DIR/corpus.src_trg_al_label"; - print STDERR "\n!!!EXTRACTING GRAMMAR\n"; - my $OUTGRAMMAR = "$GRAMMAR_DIR/grammar.bidir.gz"; - if (-e $OUTGRAMMAR) { - print STDERR "$OUTGRAMMAR exists, reusing...\n"; - } else { - my $BACKOFF_ARG = ($BACKOFF_GRAMMAR ? "-g" : ""); - safesystem("$EXTRACTOR -i $LABELED -c $ITEMS_IN_MEMORY -L $BASE_PHRASE_MAX_SIZE -b -t $NUM_TOPICS $BACKOFF_ARG | $SORT_KEYS | $REDUCER -p -b | $SORT_KEYS | $REDUCER | $GZIP > $OUTGRAMMAR") or die "Couldn't extract grammar"; - } - return $OUTGRAMMAR; -} - -sub safesystem { - print STDERR "Executing: @_\n"; - system(@_); - if ($? == -1) { - print STDERR "ERROR: Failed to execute: @_\n $!\n"; - exit(1); - } - elsif ($? & 127) { - printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - exit(1); - } - else { - my $exitcode = $? >> 8; - print STDERR "Exit code: $exitcode\n" if $exitcode; - return ! $exitcode; - } -} - diff --git a/gi/pipeline/lticluster.config b/gi/pipeline/lticluster.config deleted file mode 100644 index 3e23c8cb..00000000 --- a/gi/pipeline/lticluster.config +++ /dev/null @@ -1,9 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM dev dev-refs test1 testt-eval.sh ... -/home/cdyer/ws10smt-data -btec /home/cdyer/ws10smt-data/btec/ split.zh-en.al lm/en.3gram.lm.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -zhen /home/cdyer/ws10smt-data/chinese-english corpus.zh-en.al lm/c2e.3gram.lm.gz dev_and_test/mt02.src.txt dev_and_test/mt02.ref.* dev_and_test/mt03.src.txt eval-mt03.sh -aren /home/cdyer/ws10smt-data/arabic-english corpus.ar-en-al lm/a2e.3gram.lm.gz dev_and_test/dev.src.txt dev_and_test/dev.ref.txt.* dev_and_test/mt05.src.txt eval-mt05.sh -uren /home/cdyer/ws10smt-data/urdu-english corpus.ur-en.al lm/u2e.en.lm.gz dev/dev.ur dev/dev.en* devtest/devtest.ur eval-devtest.sh -nlfr /home/cdyer/ws10smt-data/dutch-french corpus.nl-fr.al - diff --git a/gi/pipeline/scripts/filter-by-f.pl b/gi/pipeline/scripts/filter-by-f.pl deleted file mode 100755 index 0cef0606..00000000 --- a/gi/pipeline/scripts/filter-by-f.pl +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; } - -my $REKEY="$SCRIPT_DIR/rekey.pl"; -my $REFILTER="$SCRIPT_DIR/refilter.pl"; -my $SORT="$SCRIPT_DIR/sort-by-key.sh"; -assert_exec($REKEY, $REFILTER, $SORT); - - -die "Usage: $0 NUM-TRANSLATIONS ingrammar.gz outgrammar.gz\n" unless scalar @ARGV == 3; -my $translations = shift @ARGV; -die "Need number: $translations" unless $translations > 0; -die unless $ARGV[0] =~ /\.gz$/; -die unless $ARGV[1] =~ /\.gz$/; -die if $ARGV[0] eq $ARGV[1]; -die "Can't find $ARGV[0]" unless -f $ARGV[0]; - -my $cmd = "gunzip -c $ARGV[0] | $REKEY | $SORT | $REFILTER $translations | gzip > $ARGV[1]"; -safesystem($ARGV[1], $cmd) or die "Filtering failed"; -exit 0; - -sub assert_exec { - my @files = @_; - for my $file (@files) { - die "Can't find $file - did you run make?\n" unless -e $file; - die "Can't execute $file" unless -e $file; - } -}; - -sub safesystem { - my $output = shift @_; - print STDERR "Executing: @_\n"; - system(@_); - if ($? == -1) { - print STDERR "ERROR: Failed to execute: @_\n $!\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - exit(1); - } - elsif ($? & 127) { - printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n", - ($? & 127), ($? & 128) ? 'with' : 'without'; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - exit(1); - } - else { - my $exitcode = $? >> 8; - if ($exitcode) { - print STDERR "Exit code: $exitcode\n"; - if (defined $output && -e $output) { printf STDERR "Removing $output\n"; `rm -rf $output`; } - } - return ! $exitcode; - } -} - diff --git a/gi/pipeline/scripts/patch-corpus.pl b/gi/pipeline/scripts/patch-corpus.pl deleted file mode 100755 index c0eec43e..00000000 --- a/gi/pipeline/scripts/patch-corpus.pl +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $PATCH = shift @ARGV; -my $TGT = 1; -my $APPEND; -while ($PATCH eq "-s" || $PATCH eq "-a") { - if ($PATCH eq "-s") { - undef $TGT; - } else { - $APPEND = 1; - } - $PATCH = shift @ARGV; -} - -die "Usage: $0 [-s] [-a] tagged.en[_fr] < lexical.en_fr_al[_...]\n" unless $PATCH; - -open P, "<$PATCH" or die "Can't read tagged corpus $PATCH: $!"; -my $first=<P>; close P; -my @fields = split / \|\|\| /, $first; -die "Bad format!" if (scalar @fields > 2); - -if (scalar @fields != 1) { - # TODO support this - die "Patching source and target not supported yet!"; -} - -my $line = 0; -open P, "<$PATCH" or die "Can't read tagged corpus $PATCH: $!"; -while(my $pline = <P>) { - chomp $pline; - $line++; - my $line = <>; - die "Too few lines in lexical corpus!" unless $line; - chomp $line; - @fields = split / \|\|\| /, $line; - my @pwords = split /\s+/, $pline; - if ($TGT) { - my @lwords = split /\s+/, $fields[1]; - die "Length mismatch in line $line!\n" unless (scalar @pwords == scalar @lwords); - if ($APPEND) { - foreach my $i (0..(scalar @pwords-1)) { - $lwords[$i] = $lwords[$i] . '_' . $pwords[$i]; - } - $fields[1] = join ' ', @lwords; - } else { - $fields[1] = $pline; - } - } else { # source side - my @lwords = split /\s+/, $fields[0]; - die "Length mismatch in line $line!\n" unless (scalar @pwords == scalar @lwords); - if ($APPEND) { - foreach my $i (0..(scalar @pwords-1)) { - $lwords[$i] = $lwords[$i] . '_' . $pwords[$i]; - } - $fields[0] = join ' ', @lwords; - } else { - $fields[0] = $pline; - } - } - print join ' ||| ', @fields; - print "\n"; -} - - diff --git a/gi/pipeline/scripts/refilter.pl b/gi/pipeline/scripts/refilter.pl deleted file mode 100755 index a783eb4e..00000000 --- a/gi/pipeline/scripts/refilter.pl +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -my $NUM_TRANSLATIONS = shift @ARGV; -unless ($NUM_TRANSLATIONS) { $NUM_TRANSLATIONS=30; } -print STDERR "KEEPING $NUM_TRANSLATIONS TRANSLATIONS FOR SOURCE\n"; - -my $pk = ''; -my %dict; -while(<>) { - s/^(.+)\t//; - my $key = $1; - if ($key ne $pk) { - if ($pk) { - emit_dict(); - } - %dict = (); - $pk = $key; - } - my ($lhs, $f, $e, $s) = split / \|\|\| /; - my $score = 0; - if ($s =~ /XEF=([^ ]+)/) { - $score += $1; - } else { die; } - if ($s =~ /GenerativeProb=([^ ]+)/) { - $score += ($1 / 10); - } else { die; } - $dict{"$lhs ||| $f ||| $e ||| $s"} = $score; -} -emit_dict(); - -sub emit_dict { - my $cc = 0; - for my $k (sort { $dict{$a} <=> $dict{$b} } keys %dict) { - print "$k"; - $cc++; - if ($cc >= $NUM_TRANSLATIONS) { last; } - } -} - diff --git a/gi/pipeline/scripts/rekey.pl b/gi/pipeline/scripts/rekey.pl deleted file mode 100755 index 31eb86b8..00000000 --- a/gi/pipeline/scripts/rekey.pl +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/perl - -while(<>) { - my ($lhs, $f, $e, $s) = split / \|\|\| /; - $f =~ s/\[X[0-9]+\]/\[X\]/g; - print "$f\t$_"; -} - diff --git a/gi/pipeline/scripts/remove-tags-from-contexts.pl b/gi/pipeline/scripts/remove-tags-from-contexts.pl deleted file mode 100755 index 20698816..00000000 --- a/gi/pipeline/scripts/remove-tags-from-contexts.pl +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -use Getopt::Long "GetOptions"; - -my $PHRASE = 'tok'; -my $CONTEXT = 'tag'; - -die "Usage: $0 [--phrase=tok|tag] [--context=tok|tag] < corpus" - unless &GetOptions('phrase=s' => \$PHRASE, 'context=s' => \$CONTEXT); - -my $lno = 0; -while(my $line = <>) { - $lno++; - chomp $line; - my @top = split /\t/, $line; - die unless (scalar @top == 2); - - my @pwords = split /\s+/, $top[0]; - foreach my $token (@pwords) { - #print $token . "\n"; - my @parts = split /_(?!.*_)/, $token; - die unless (scalar @parts == 2); - if ($PHRASE eq "tok") { - $token = $parts[0] - } elsif ($PHRASE eq "tag") { - $token = $parts[1] - } - } - - my @fields = split / \|\|\| /, $top[1]; - foreach my $i (0..((scalar @fields) / 2 - 1)) { - #print $i . ": " . $fields[2*$i] . " of " . (scalar @fields) . "\n"; - my @cwords = split /\s+/, $fields[2*$i]; - foreach my $token (@cwords) { - #print $i . ": " . $token . "\n"; - my @parts = split /_(?!.*_)/, $token; - if (scalar @parts == 2) { - if ($CONTEXT eq "tok") { - $token = $parts[0] - } elsif ($CONTEXT eq "tag") { - $token = $parts[1] - } - } - } - $fields[2*$i] = join ' ', @cwords; - } - - print join ' ', @pwords; - print "\t"; - print join ' ||| ', @fields; - print "\n"; -} diff --git a/gi/pipeline/scripts/remove-tags-from-corpus.pl b/gi/pipeline/scripts/remove-tags-from-corpus.pl deleted file mode 100755 index be3e97c0..00000000 --- a/gi/pipeline/scripts/remove-tags-from-corpus.pl +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -use Getopt::Long "GetOptions"; - -my $LANGUAGE = shift @ARGV; -$LANGUAGE = 'target' unless ($LANGUAGE); - -my $lno = 0; -while(my $line = <>) { - $lno++; - chomp $line; - - my @fields = split / \|\|\| /, $line; - - if ($LANGUAGE eq "source" or $LANGUAGE eq "both") { - my @cwords = split /\s+/, $fields[0]; - foreach my $token (@cwords) { - my @parts = split /_(?!.*_)/, $token; - if (scalar @parts == 2) { - $token = $parts[0] - } else { - print STDERR "WARNING: invalid tagged token $token\n"; - } - } - $fields[0] = join ' ', @cwords; - } - - if ($LANGUAGE eq "target" or $LANGUAGE eq "both") { - my @cwords = split /\s+/, $fields[1]; - foreach my $token (@cwords) { - my @parts = split /_(?!.*_)/, $token; - if (scalar @parts == 2) { - $token = $parts[1] - } else { - print STDERR "WARNING: invalid tagged token $token\n"; - } - } - $fields[0] = join ' ', @cwords; - } - - print join ' ||| ', @fields; - print "\n"; -} diff --git a/gi/pipeline/scripts/sort-by-key.sh b/gi/pipeline/scripts/sort-by-key.sh deleted file mode 100755 index 7ae33e03..00000000 --- a/gi/pipeline/scripts/sort-by-key.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -export LANG=C -sort -t $'\t' -k 1 -T /tmp -S 6000000000 - diff --git a/gi/pipeline/scripts/xfeats.pl b/gi/pipeline/scripts/xfeats.pl deleted file mode 100755 index dc578513..00000000 --- a/gi/pipeline/scripts/xfeats.pl +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -die "Usage: $0 x-grammar.scfg[.gz] < cat-grammar.scfg\n" unless scalar @ARGV > 0; - -my $xgrammar = shift @ARGV; -die "Can't find $xgrammar" unless -f $xgrammar; -my $fh; -if ($xgrammar =~ /\.gz$/) { - open $fh, "gunzip -c $xgrammar|" or die "Can't fork: $!"; -} else { - open $fh, "<$xgrammar" or die "Can't read $xgrammar: $!"; -} -print STDERR "Reading X-feats from $xgrammar...\n"; -my %dict; -while(<$fh>) { - chomp; - my ($lhs, $f, $e, $feats) = split / \|\|\| /; - my $xfeats; - my $cc = 0; - my @xfeats = (); - while ($feats =~ /(EGivenF|FGivenE|LogRuleCount|LogECount|LogFCount|SingletonRule|SingletonE|SingletonF)=([^ ]+)( |$)/og) { - push @xfeats, "X_$1=$2"; - } - #print "$lhs ||| $f ||| $e ||| @xfeats\n"; - $dict{"$lhs ||| $f ||| $e"} = "@xfeats"; -} -close $fh; - -print STDERR "Add features...\n"; -while(<>) { - chomp; - my ($lhs, $f, $e) = split / \|\|\| /; - $f=~ s/\[[^]]+,([12])\]/\[X,$1\]/g; - my $xfeats = $dict{"[X] ||| $f ||| $e"}; - die "Can't find x features for: $_\n" unless $xfeats; - print "$_ $xfeats\n"; -} - diff --git a/gi/pipeline/valhalla.config b/gi/pipeline/valhalla.config deleted file mode 100644 index e00a8485..00000000 --- a/gi/pipeline/valhalla.config +++ /dev/null @@ -1,9 +0,0 @@ -# THIS FILE GIVES THE LOCATIONS OF THE CORPORA USED -# name path aligned-corpus LM dev dev-refs test1 testt-eval.sh ... -/home/chris/ws10smt/data -btec /home/chris/ws10smt/data/btec/ split.zh-en.al lm/en.3gram.lm.gz devtest/devset1_2.zh devtest/devset1_2.lc.en* devtest/devset3.zh eval-devset3.sh -fbis /home/chris/ws10smt/data/chinese-english.fbis corpus.zh-en.al -zhen /home/chris/ws10smt/data/chinese-english corpus.zh-en.al -aren /home/chris/ws10smt/data/arabic-english corpus.ar-en.al -uren /home/chris/ws10smt/data/urdu-english corpus.ur-en.al lm/u2e.en.lm.gz dev/dev.ur dev/dev.en* devtest/devtest.ur eval-devtest.sh -nlfr /home/chris/ws10smt/data/dutch-french corpus.nl-fr.al diff --git a/gi/posterior-regularisation/Corpus.java b/gi/posterior-regularisation/Corpus.java deleted file mode 100644 index 07b27387..00000000 --- a/gi/posterior-regularisation/Corpus.java +++ /dev/null @@ -1,167 +0,0 @@ -import gnu.trove.TIntArrayList; - -import java.io.*; -import java.util.*; -import java.util.regex.Pattern; - -public class Corpus -{ - private Lexicon<String> tokenLexicon = new Lexicon<String>(); - private Lexicon<TIntArrayList> phraseLexicon = new Lexicon<TIntArrayList>(); - private Lexicon<TIntArrayList> contextLexicon = new Lexicon<TIntArrayList>(); - private List<Edge> edges = new ArrayList<Edge>(); - private List<List<Edge>> phraseToContext = new ArrayList<List<Edge>>(); - private List<List<Edge>> contextToPhrase = new ArrayList<List<Edge>>(); - - public class Edge - { - Edge(int phraseId, int contextId, int count) - { - this.phraseId = phraseId; - this.contextId = contextId; - this.count = count; - } - public int getPhraseId() - { - return phraseId; - } - public TIntArrayList getPhrase() - { - return phraseLexicon.lookup(phraseId); - } - public String getPhraseString() - { - StringBuffer b = new StringBuffer(); - for (int tid: getPhrase().toNativeArray()) - { - if (b.length() > 0) - b.append(" "); - b.append(tokenLexicon.lookup(tid)); - } - return b.toString(); - } - public int getContextId() - { - return contextId; - } - public TIntArrayList getContext() - { - return contextLexicon.lookup(contextId); - } - public String getContextString() - { - StringBuffer b = new StringBuffer(); - for (int tid: getContext().toNativeArray()) - { - if (b.length() > 0) - b.append(" "); - b.append(tokenLexicon.lookup(tid)); - } - return b.toString(); - } - public int getCount() - { - return count; - } - private int phraseId; - private int contextId; - private int count; - } - - List<Edge> getEdges() - { - return edges; - } - - int getNumEdges() - { - return edges.size(); - } - - int getNumPhrases() - { - return phraseLexicon.size(); - } - - List<Edge> getEdgesForPhrase(int phraseId) - { - return phraseToContext.get(phraseId); - } - - int getNumContexts() - { - return contextLexicon.size(); - } - - List<Edge> getEdgesForContext(int contextId) - { - return contextToPhrase.get(contextId); - } - - int getNumTokens() - { - return tokenLexicon.size(); - } - - static Corpus readFromFile(Reader in) throws IOException - { - Corpus c = new Corpus(); - - // read in line-by-line - BufferedReader bin = new BufferedReader(in); - String line; - Pattern separator = Pattern.compile(" \\|\\|\\| "); - - while ((line = bin.readLine()) != null) - { - // split into phrase and contexts - StringTokenizer st = new StringTokenizer(line, "\t"); - assert (st.hasMoreTokens()); - String phraseToks = st.nextToken(); - assert (st.hasMoreTokens()); - String rest = st.nextToken(); - assert (!st.hasMoreTokens()); - - // process phrase - st = new StringTokenizer(phraseToks, " "); - TIntArrayList ptoks = new TIntArrayList(); - while (st.hasMoreTokens()) - ptoks.add(c.tokenLexicon.insert(st.nextToken())); - int phraseId = c.phraseLexicon.insert(ptoks); - if (phraseId == c.phraseToContext.size()) - c.phraseToContext.add(new ArrayList<Edge>()); - - // process contexts - String[] parts = separator.split(rest); - assert (parts.length % 2 == 0); - for (int i = 0; i < parts.length; i += 2) - { - // process pairs of strings - context and count - TIntArrayList ctx = new TIntArrayList(); - String ctxString = parts[i]; - String countString = parts[i + 1]; - StringTokenizer ctxStrtok = new StringTokenizer(ctxString, " "); - while (ctxStrtok.hasMoreTokens()) - { - String token = ctxStrtok.nextToken(); - if (!token.equals("<PHRASE>")) - ctx.add(c.tokenLexicon.insert(token)); - } - int contextId = c.contextLexicon.insert(ctx); - if (contextId == c.contextToPhrase.size()) - c.contextToPhrase.add(new ArrayList<Edge>()); - - assert (countString.startsWith("C=")); - Edge e = c.new Edge(phraseId, contextId, - Integer.parseInt(countString.substring(2).trim())); - c.edges.add(e); - - // index the edge for fast phrase, context lookup - c.phraseToContext.get(phraseId).add(e); - c.contextToPhrase.get(contextId).add(e); - } - } - - return c; - } -} diff --git a/gi/posterior-regularisation/Lexicon.java b/gi/posterior-regularisation/Lexicon.java deleted file mode 100644 index 9f0245ee..00000000 --- a/gi/posterior-regularisation/Lexicon.java +++ /dev/null @@ -1,32 +0,0 @@ -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class Lexicon<T> -{ - public int insert(T word) - { - Integer i = wordToIndex.get(word); - if (i == null) - { - i = indexToWord.size(); - wordToIndex.put(word, i); - indexToWord.add(word); - } - return i; - } - - public T lookup(int index) - { - return indexToWord.get(index); - } - - public int size() - { - return indexToWord.size(); - } - - private Map<T, Integer> wordToIndex = new HashMap<T, Integer>(); - private List<T> indexToWord = new ArrayList<T>(); -}
\ No newline at end of file diff --git a/gi/posterior-regularisation/PhraseContextModel.java b/gi/posterior-regularisation/PhraseContextModel.java deleted file mode 100644 index 85bcfb89..00000000 --- a/gi/posterior-regularisation/PhraseContextModel.java +++ /dev/null @@ -1,466 +0,0 @@ -// Input of the form: -// " the phantom of the opera " tickets for <PHRASE> tonight ? ||| C=1 ||| seats for <PHRASE> ? </s> ||| C=1 ||| i see <PHRASE> ? </s> ||| C=1 -// phrase TAB [context]+ -// where context = phrase ||| C=... which are separated by ||| - -// Model parameterised as follows: -// - each phrase, p, is allocated a latent state, t -// - this is used to generate the contexts, c -// - each context is generated using 4 independent multinomials, one for each position LL, L, R, RR - -// Training with EM: -// - e-step is estimating q(t) = P(t|p,c) for all x,c -// - m-step is estimating model parameters P(c,t|p) = P(t) P(c|t) -// - PR uses alternate e-step, which first optimizes lambda -// min_q KL(q||p) + delta sum_pt max_c E_q[phi_ptc] -// where -// q(t|p,c) propto p(t,c|p) exp( -phi_ptc ) -// Then q is used to obtain expectations for vanilla M-step. - -// Sexing it up: -// - learn p-specific conditionals P(t|p) -// - or generate phrase internals, e.g., generate edge words from -// different distribution to central words -// - agreement between phrase->context model and context->phrase model - -import java.io.*; -import optimization.gradientBasedMethods.*; -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.gradientBasedMethods.stats.ProjectedOptimizerStats; -import optimization.linesearch.ArmijoLineSearchMinimizationAlongProjectionArc; -import optimization.linesearch.GenericPickFirstStep; -import optimization.linesearch.InterpolationPickFirstStep; -import optimization.linesearch.LineSearchMethod; -import optimization.linesearch.WolfRuleLineSearch; -import optimization.projections.SimplexProjection; -import optimization.stopCriteria.CompositeStopingCriteria; -import optimization.stopCriteria.NormalizedProjectedGradientL2Norm; -import optimization.stopCriteria.NormalizedValueDifference; -import optimization.stopCriteria.ProjectedGradientL2Norm; -import optimization.stopCriteria.StopingCriteria; -import optimization.stopCriteria.ValueDifference; -import optimization.util.MathUtils; -import java.util.*; -import java.util.regex.*; -import gnu.trove.TDoubleArrayList; -import gnu.trove.TIntArrayList; -import static java.lang.Math.*; - -class PhraseContextModel -{ - // model/optimisation configuration parameters - int numTags; - boolean posteriorRegularisation = true; - double constraintScale = 3; // FIXME: make configurable - - // copied from L1LMax in depparsing code - final double c1= 0.0001, c2=0.9, stoppingPrecision = 1e-5, maxStep = 10; - final int maxZoomEvals = 10, maxExtrapolationIters = 200; - int maxProjectionIterations = 200; - int minOccurrencesForProjection = 0; - - // book keeping - int numPositions; - Random rng = new Random(); - - // training set - Corpus training; - - // model parameters (learnt) - double emissions[][][]; // position in 0 .. 3 x tag x word Pr(word | tag, position) - double prior[][]; // phrase x tag Pr(tag | phrase) - double lambda[]; // edge = (phrase, context) x tag flattened lagrange multipliers - - PhraseContextModel(Corpus training, int tags) - { - this.training = training; - this.numTags = tags; - assert (!training.getEdges().isEmpty()); - assert (numTags > 1); - - // now initialise emissions - numPositions = training.getEdges().get(0).getContext().size(); - assert (numPositions > 0); - - emissions = new double[numPositions][numTags][training.getNumTokens()]; - prior = new double[training.getNumEdges()][numTags]; - if (posteriorRegularisation) - lambda = new double[training.getNumEdges() * numTags]; - - for (double[][] emissionTW : emissions) - { - for (double[] emissionW : emissionTW) - { - randomise(emissionW); -// for (int i = 0; i < emissionW.length; ++i) -// emissionW[i] = i+1; -// normalise(emissionW); - } - } - - for (double[] priorTag : prior) - { - randomise(priorTag); -// for (int i = 0; i < priorTag.length; ++i) -// priorTag[i] = i+1; -// normalise(priorTag); - } - } - - void expectationMaximisation(int numIterations) - { - double lastLlh = Double.NEGATIVE_INFINITY; - - for (int iteration = 0; iteration < numIterations; ++iteration) - { - double emissionsCounts[][][] = new double[numPositions][numTags][training.getNumTokens()]; - double priorCounts[][] = new double[training.getNumPhrases()][numTags]; - - // E-step - double llh = 0; - if (posteriorRegularisation) - { - EStepDualObjective objective = new EStepDualObjective(); - - // copied from x2y2withconstraints -// LineSearchMethod ls = new ArmijoLineSearchMinimizationAlongProjectionArc(new InterpolationPickFirstStep(1)); -// OptimizerStats stats = new OptimizerStats(); -// ProjectedGradientDescent optimizer = new ProjectedGradientDescent(ls); -// CompositeStopingCriteria compositeStop = new CompositeStopingCriteria(); -// compositeStop.add(new ProjectedGradientL2Norm(0.001)); -// compositeStop.add(new ValueDifference(0.001)); -// optimizer.setMaxIterations(50); -// boolean succeed = optimizer.optimize(objective,stats,compositeStop); - - // copied from depparser l1lmaxobjective - ProjectedOptimizerStats stats = new ProjectedOptimizerStats(); - GenericPickFirstStep pickFirstStep = new GenericPickFirstStep(1); - LineSearchMethod linesearch = new WolfRuleLineSearch(pickFirstStep, c1, c2); - ProjectedGradientDescent optimizer = new ProjectedGradientDescent(linesearch); - optimizer.setMaxIterations(maxProjectionIterations); - CompositeStopingCriteria stop = new CompositeStopingCriteria(); - stop.add(new NormalizedProjectedGradientL2Norm(stoppingPrecision)); - stop.add(new NormalizedValueDifference(stoppingPrecision)); - boolean succeed = optimizer.optimize(objective, stats, stop); - - System.out.println("Ended optimzation Projected Gradient Descent\n" + stats.prettyPrint(1)); - //System.out.println("Solution: " + objective.parameters); - if (!succeed) - System.out.println("Failed to optimize"); - //System.out.println("Ended optimization in " + optimizer.getCurrentIteration()); - - //lambda = objective.getParameters(); - llh = objective.primal(); - - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List<Corpus.Edge> edges = training.getEdgesForPhrase(i); - for (int j = 0; j < edges.size(); ++j) - { - Corpus.Edge e = edges.get(j); - for (int t = 0; t < numTags; t++) - { - double p = objective.q.get(i).get(j).get(t); - priorCounts[i][t] += e.getCount() * p; - TIntArrayList tokens = e.getContext(); - for (int k = 0; k < tokens.size(); ++k) - emissionsCounts[k][t][tokens.get(k)] += e.getCount() * p; - } - } - } - } - else - { - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List<Corpus.Edge> edges = training.getEdgesForPhrase(i); - for (int j = 0; j < edges.size(); ++j) - { - Corpus.Edge e = edges.get(j); - double probs[] = posterior(i, e); - double z = normalise(probs); - llh += log(z) * e.getCount(); - - TIntArrayList tokens = e.getContext(); - for (int t = 0; t < numTags; ++t) - { - priorCounts[i][t] += e.getCount() * probs[t]; - for (int k = 0; k < tokens.size(); ++k) - emissionsCounts[j][t][tokens.get(k)] += e.getCount() * probs[t]; - } - } - } - } - - // M-step: normalise - for (double[][] emissionTW : emissionsCounts) - for (double[] emissionW : emissionTW) - normalise(emissionW); - - for (double[] priorTag : priorCounts) - normalise(priorTag); - - emissions = emissionsCounts; - prior = priorCounts; - - System.out.println("Iteration " + iteration + " llh " + llh); - -// if (llh - lastLlh < 1e-4) -// break; -// else -// lastLlh = llh; - } - } - - static double normalise(double probs[]) - { - double z = 0; - for (double p : probs) - z += p; - for (int i = 0; i < probs.length; ++i) - probs[i] /= z; - return z; - } - - void randomise(double probs[]) - { - double z = 0; - for (int i = 0; i < probs.length; ++i) - { - probs[i] = 10 + rng.nextDouble(); - z += probs[i]; - } - - for (int i = 0; i < probs.length; ++i) - probs[i] /= z; - } - - static int argmax(double probs[]) - { - double m = Double.NEGATIVE_INFINITY; - int mi = -1; - for (int i = 0; i < probs.length; ++i) - { - if (probs[i] > m) - { - m = probs[i]; - mi = i; - } - } - return mi; - } - - double[] posterior(int phraseId, Corpus.Edge e) // unnormalised - { - double probs[] = new double[numTags]; - TIntArrayList tokens = e.getContext(); - for (int t = 0; t < numTags; ++t) - { - probs[t] = prior[phraseId][t]; - for (int k = 0; k < tokens.size(); ++k) - probs[t] *= emissions[k][t][tokens.get(k)]; - } - return probs; - } - - void displayPosterior() - { - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List<Corpus.Edge> edges = training.getEdgesForPhrase(i); - for (Corpus.Edge e: edges) - { - double probs[] = posterior(i, e); - normalise(probs); - - // emit phrase - System.out.print(e.getPhraseString()); - System.out.print("\t"); - System.out.print(e.getContextString()); - System.out.print("||| C=" + e.getCount() + " |||"); - - int t = argmax(probs); - System.out.print(" " + t + " ||| " + probs[t]); - // for (int t = 0; t < numTags; ++t) - // System.out.print(" " + probs[t]); - System.out.println(); - } - } - } - - public static void main(String[] args) - { - assert (args.length >= 2); - try - { - Corpus corpus = Corpus.readFromFile(new FileReader(new File(args[0]))); - PhraseContextModel model = new PhraseContextModel(corpus, Integer.parseInt(args[1])); - model.expectationMaximisation(Integer.parseInt(args[2])); - model.displayPosterior(); - } - catch (IOException e) - { - System.out.println("Failed to read input file: " + args[0]); - e.printStackTrace(); - } - } - - class EStepDualObjective extends ProjectedObjective - { - List<List<TDoubleArrayList>> conditionals; // phrase id x context # x tag - precomputed - List<List<TDoubleArrayList>> q; // ditto, but including exp(-lambda) terms - double objective = 0; // log(z) - // Objective.gradient = d log(z) / d lambda = E_q[phi] - double llh = 0; - - public EStepDualObjective() - { - super(); - // compute conditionals p(context, tag | phrase) for all training instances - conditionals = new ArrayList<List<TDoubleArrayList>>(training.getNumPhrases()); - q = new ArrayList<List<TDoubleArrayList>>(training.getNumPhrases()); - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List<Corpus.Edge> edges = training.getEdgesForPhrase(i); - - conditionals.add(new ArrayList<TDoubleArrayList>(edges.size())); - q.add(new ArrayList<TDoubleArrayList>(edges.size())); - - for (int j = 0; j < edges.size(); ++j) - { - Corpus.Edge e = edges.get(j); - double probs[] = posterior(i, e); - double z = normalise(probs); - llh += log(z) * e.getCount(); - conditionals.get(i).add(new TDoubleArrayList(probs)); - q.get(i).add(new TDoubleArrayList(probs)); - } - } - - gradient = new double[training.getNumEdges()*numTags]; - setInitialParameters(lambda); - computeObjectiveAndGradient(); - } - - @Override - public double[] projectPoint(double[] point) - { - SimplexProjection p = new SimplexProjection(constraintScale); - - double[] newPoint = point.clone(); - int edgeIndex = 0; - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List<Corpus.Edge> edges = training.getEdgesForPhrase(i); - - for (int t = 0; t < numTags; t++) - { - double[] subPoint = new double[edges.size()]; - for (int j = 0; j < edges.size(); ++j) - subPoint[j] = point[edgeIndex+j*numTags+t]; - - p.project(subPoint); - for (int j = 0; j < edges.size(); ++j) - newPoint[edgeIndex+j*numTags+t] = subPoint[j]; - } - - edgeIndex += edges.size() * numTags; - } -// System.out.println("Proj from: " + Arrays.toString(point)); -// System.out.println("Proj to: " + Arrays.toString(newPoint)); - return newPoint; - } - - @Override - public void setParameters(double[] params) - { - super.setParameters(params); - computeObjectiveAndGradient(); - } - - @Override - public double[] getGradient() - { - gradientCalls += 1; - return gradient; - } - - @Override - public double getValue() - { - functionCalls += 1; - return objective; - } - - public void computeObjectiveAndGradient() - { - int edgeIndex = 0; - objective = 0; - Arrays.fill(gradient, 0); - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List<Corpus.Edge> edges = training.getEdgesForPhrase(i); - - for (int j = 0; j < edges.size(); ++j) - { - Corpus.Edge e = edges.get(j); - - double z = 0; - for (int t = 0; t < numTags; t++) - { - double v = conditionals.get(i).get(j).get(t) * exp(-parameters[edgeIndex+t]); - q.get(i).get(j).set(t, v); - z += v; - } - objective += log(z) * e.getCount(); - - for (int t = 0; t < numTags; t++) - { - double v = q.get(i).get(j).get(t) / z; - q.get(i).get(j).set(t, v); - gradient[edgeIndex+t] -= e.getCount() * v; - } - - edgeIndex += numTags; - } - } -// System.out.println("computeObjectiveAndGradient logz=" + objective); -// System.out.println("lambda= " + Arrays.toString(parameters)); -// System.out.println("gradient=" + Arrays.toString(gradient)); - } - - public String toString() - { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getCanonicalName()).append(" with "); - sb.append(parameters.length).append(" parameters and "); - sb.append(training.getNumPhrases() * numTags).append(" constraints"); - return sb.toString(); - } - - double primal() - { - // primal = llh + KL(q||p) + scale * sum_pt max_c E_q[phi_pct] - // kl = sum_Y q(Y) log q(Y) / p(Y|X) - // = sum_Y q(Y) { -lambda . phi(Y) - log Z } - // = -log Z - lambda . E_q[phi] - // = -objective + lambda . gradient - - double kl = -objective + MathUtils.dotProduct(parameters, gradient); - double l1lmax = 0; - for (int i = 0; i < training.getNumPhrases(); ++i) - { - List<Corpus.Edge> edges = training.getEdgesForPhrase(i); - for (int t = 0; t < numTags; t++) - { - double lmax = Double.NEGATIVE_INFINITY; - for (int j = 0; j < edges.size(); ++j) - lmax = max(lmax, q.get(i).get(j).get(t)); - l1lmax += lmax; - } - } - - return llh + kl + constraintScale * l1lmax; - } - } -} diff --git a/gi/posterior-regularisation/README b/gi/posterior-regularisation/README deleted file mode 100644 index a3d54ffc..00000000 --- a/gi/posterior-regularisation/README +++ /dev/null @@ -1,3 +0,0 @@ - 557 ./cdec_extools/extractor -i btec/split.zh-en.al -c 500000 -L 12 -C | sort -t $'\t' -k 1 | ./cdec_extools/mr_stripe_rule_reduce > btec.concordance - 559 wc -l btec.concordance - 588 cat btec.concordance | sed 's/.* //' | awk '{ for (i=1; i < NF; i++) { x=substr($i, 1, 2); if (x == "C=") printf "\n"; else if (x != "||") printf "%s ", $i; }; printf "\n"; }' | sort | uniq | wc -l diff --git a/gi/posterior-regularisation/alphabet.hh b/gi/posterior-regularisation/alphabet.hh deleted file mode 100644 index 1db928da..00000000 --- a/gi/posterior-regularisation/alphabet.hh +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef _alphabet_hh -#define _alphabet_hh - -#include <cassert> -#include <iosfwd> -#include <map> -#include <string> -#include <vector> - -// Alphabet: indexes a set of types -template <typename T> -class Alphabet: protected std::map<T, int> -{ -public: - Alphabet() {}; - - bool empty() const { return std::map<T,int>::empty(); } - int size() const { return std::map<T,int>::size(); } - - int operator[](const T &k) const - { - typename std::map<T,int>::const_iterator cit = find(k); - if (cit != std::map<T,int>::end()) - return cit->second; - else - return -1; - } - - int lookup(const T &k) const { return (*this)[k]; } - - int insert(const T &k) - { - int sz = size(); - assert((unsigned) sz == _items.size()); - - std::pair<typename std::map<T,int>::iterator, bool> - ins = std::map<T,int>::insert(make_pair(k, sz)); - - if (ins.second) - _items.push_back(k); - - return ins.first->second; - } - - const T &type(int i) const - { - assert(i >= 0); - assert(i < size()); - return _items[i]; - } - - std::ostream &display(std::ostream &out, int i) const - { - return out << type(i); - } - -private: - std::vector<T> _items; -}; - -#endif diff --git a/gi/posterior-regularisation/canned.concordance b/gi/posterior-regularisation/canned.concordance deleted file mode 100644 index 710973ff..00000000 --- a/gi/posterior-regularisation/canned.concordance +++ /dev/null @@ -1,4 +0,0 @@ -a 0 0 <PHRASE> 0 0 ||| C=1 ||| 1 1 <PHRASE> 1 1 ||| C=1 ||| 2 2 <PHRASE> 2 2 ||| C=1 -b 0 0 <PHRASE> 0 0 ||| C=1 ||| 1 1 <PHRASE> 1 1 ||| C=1 -c 2 2 <PHRASE> 2 2 ||| C=1 ||| 4 4 <PHRASE> 4 4 ||| C=1 ||| 5 5 <PHRASE> 5 5 ||| C=1 -d 4 4 <PHRASE> 4 4 ||| C=1 ||| 5 5 <PHRASE> 5 5 ||| C=1 diff --git a/gi/posterior-regularisation/em.cc b/gi/posterior-regularisation/em.cc deleted file mode 100644 index f6c9fd68..00000000 --- a/gi/posterior-regularisation/em.cc +++ /dev/null @@ -1,830 +0,0 @@ -// Input of the form: -// " the phantom of the opera " tickets for <PHRASE> tonight ? ||| C=1 ||| seats for <PHRASE> ? </s> ||| C=1 ||| i see <PHRASE> ? </s> ||| C=1 -// phrase TAB [context]+ -// where context = phrase ||| C=... which are separated by ||| - -// Model parameterised as follows: -// - each phrase, p, is allocated a latent state, t -// - this is used to generate the contexts, c -// - each context is generated using 4 independent multinomials, one for each position LL, L, R, RR - -// Training with EM: -// - e-step is estimating P(t|p,c) for all x,c -// - m-step is estimating model parameters P(p,c,t) = P(t) P(p|t) P(c|t) - -// Sexing it up: -// - constrain the posteriors P(t|c) and P(t|p) to have few high-magnitude entries -// - improve the generation of phrase internals, e.g., generate edge words from -// different distribution to central words - -#include "alphabet.hh" -#include "log_add.hh" -#include <algorithm> -#include <fstream> -#include <iostream> -#include <iterator> -#include <map> -#include <sstream> -#include <stdexcept> -#include <vector> -#include <tr1/random> -#include <tr1/tuple> -#include <nlopt.h> - -using namespace std; -using namespace std::tr1; - -const int numTags = 5; -const int numIterations = 100; -const bool posterior_regularisation = true; -const double PHRASE_VIOLATION_WEIGHT = 10; -const double CONTEXT_VIOLATION_WEIGHT = 0; -const bool includePhraseProb = false; - -// Data structures: -Alphabet<string> lexicon; -typedef vector<int> Phrase; -typedef tuple<int, int, int, int> Context; -Alphabet<Phrase> phrases; -Alphabet<Context> contexts; - -typedef map<int, int> ContextCounts; -typedef map<int, int> PhraseCounts; -typedef map<int, ContextCounts> PhraseToContextCounts; -typedef map<int, PhraseCounts> ContextToPhraseCounts; - -PhraseToContextCounts concordancePhraseToContexts; -ContextToPhraseCounts concordanceContextToPhrases; - -typedef vector<double> Dist; -typedef vector<Dist> ConditionalDist; -Dist prior; // class -> P(class) -vector<ConditionalDist> probCtx; // word -> class -> P(word | class), for each position of context word -ConditionalDist probPhrase; // class -> P(word | class) -Dist probPhraseLength; // class -> P(length | class) expressed as geometric distribution parameter - -mt19937 randomGenerator((size_t) time(NULL)); -uniform_real<double> uniDist(0.0, 1e-1); -variate_generator< mt19937, uniform_real<double> > rng(randomGenerator, uniDist); - -void addRandomNoise(Dist &d); -void normalise(Dist &d); -void addTo(Dist &d, const Dist &e); -int argmax(const Dist &d); - -map<Phrase, map<Context, int> > lambda_indices; - -Dist conditional_probs(const Phrase &phrase, const Context &context, double *normalisation = 0); -template <typename T> -Dist -penalised_conditionals(const Phrase &phrase, const Context &context, - const T &lambda, double *normalisation); -//Dist penalised_conditionals(const Phrase &phrase, const Context &context, const double *lambda, double *normalisation = 0); -double penalised_log_likelihood(int n, const double *lambda, double *gradient, void *data); -void optimise_lambda(double delta, double gamma, vector<double> &lambda); -double expected_violation_phrases(const double *lambda); -double expected_violation_contexts(const double *lambda); -double primal_kl_divergence(const double *lambda); -double dual(const double *lambda); -void print_primal_dual(const double *lambda, double delta, double gamma); - -ostream &operator<<(ostream &, const Phrase &); -ostream &operator<<(ostream &, const Context &); -ostream &operator<<(ostream &, const Dist &); -ostream &operator<<(ostream &, const ConditionalDist &); - -int -main(int argc, char *argv[]) -{ - randomGenerator.seed(time(NULL)); - - int edges = 0; - istream &input = cin; - while (input.good()) - { - // read the phrase - string phraseString; - Phrase phrase; - getline(input, phraseString, '\t'); - istringstream pinput(phraseString); - string token; - while (pinput >> token) - phrase.push_back(lexicon.insert(token)); - int phraseId = phrases.insert(phrase); - - // read the rest, storing each context - string remainder; - getline(input, remainder, '\n'); - istringstream rinput(remainder); - Context context(-1, -1, -1, -1); - int index = 0; - while (rinput >> token) - { - if (token != "|||" && token != "<PHRASE>") - { - if (index < 4) - { - // eugh! damn templates - switch (index) - { - case 0: get<0>(context) = lexicon.insert(token); break; - case 1: get<1>(context) = lexicon.insert(token); break; - case 2: get<2>(context) = lexicon.insert(token); break; - case 3: get<3>(context) = lexicon.insert(token); break; - default: assert(false); - } - index += 1; - } - else if (token.find("C=") == 0) - { - int contextId = contexts.insert(context); - int count = atoi(token.substr(strlen("C=")).c_str()); - concordancePhraseToContexts[phraseId][contextId] += count; - concordanceContextToPhrases[contextId][phraseId] += count; - index = 0; - context = Context(-1, -1, -1, -1); - edges += 1; - } - } - } - - // trigger EOF - input >> ws; - } - - cout << "Read in " << phrases.size() << " phrases" - << " and " << contexts.size() << " contexts" - << " and " << edges << " edges" - << " and " << lexicon.size() << " word types\n"; - - // FIXME: filter out low count phrases and low count contexts (based on individual words?) - // now populate model parameters with uniform + random noise - prior.resize(numTags, 1.0); - addRandomNoise(prior); - normalise(prior); - - probCtx.resize(4, ConditionalDist(numTags, Dist(lexicon.size(), 1.0))); - if (includePhraseProb) - probPhrase.resize(numTags, Dist(lexicon.size(), 1.0)); - for (int t = 0; t < numTags; ++t) - { - for (int j = 0; j < 4; ++j) - { - addRandomNoise(probCtx[j][t]); - normalise(probCtx[j][t]); - } - if (includePhraseProb) - { - addRandomNoise(probPhrase[t]); - normalise(probPhrase[t]); - } - } - if (includePhraseProb) - { - probPhraseLength.resize(numTags, 0.5); // geometric distribution p=0.5 - addRandomNoise(probPhraseLength); - } - - cout << "\tprior: " << prior << "\n"; - //cout << "\tcontext: " << probCtx << "\n"; - //cout << "\tphrase: " << probPhrase << "\n"; - //cout << "\tphraseLen: " << probPhraseLength << endl; - - vector<double> lambda; - - // now do EM training - for (int iteration = 0; iteration < numIterations; ++iteration) - { - cout << "EM iteration " << iteration << endl; - - if (posterior_regularisation) - optimise_lambda(PHRASE_VIOLATION_WEIGHT, CONTEXT_VIOLATION_WEIGHT, lambda); - //cout << "\tlambda " << lambda << endl; - - Dist countsPrior(numTags, 0.0); - vector<ConditionalDist> countsCtx(4, ConditionalDist(numTags, Dist(lexicon.size(), 1e-10))); - ConditionalDist countsPhrase(numTags, Dist(lexicon.size(), 1e-10)); - Dist countsPhraseLength(numTags, 0.0); - Dist nPhrases(numTags, 0.0); - - double llh = 0; - for (PhraseToContextCounts::iterator pcit = concordancePhraseToContexts.begin(); - pcit != concordancePhraseToContexts.end(); ++pcit) - { - const Phrase &phrase = phrases.type(pcit->first); - - // e-step: estimate latent class probs; compile (class,word) stats for m-step - for (ContextCounts::iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - - double z = 0; - Dist tagCounts; - if (!posterior_regularisation) - tagCounts = conditional_probs(phrase, context, &z); - else - tagCounts = penalised_conditionals(phrase, context, lambda, &z); - - llh += log(z) * ccit->second; - addTo(countsPrior, tagCounts); // FIXME: times ccit->secon - - for (int t = 0; t < numTags; ++t) - { - for (int j = 0; j < 4; ++j) - countsCtx[j][t][get<0>(context)] += tagCounts[t] * ccit->second; - - if (includePhraseProb) - { - for (Phrase::const_iterator pit = phrase.begin(); pit != phrase.end(); ++pit) - countsPhrase[t][*pit] += tagCounts[t] * ccit->second; - countsPhraseLength[t] += phrase.size() * tagCounts[t] * ccit->second; - nPhrases[t] += tagCounts[t] * ccit->second; - } - } - } - } - - cout << "M-step\n"; - - // m-step: normalise prior and (class,word) stats and assign to model parameters - normalise(countsPrior); - prior = countsPrior; - for (int t = 0; t < numTags; ++t) - { - //cout << "\t\tt " << t << " prior " << countsPrior[t] << "\n"; - for (int j = 0; j < 4; ++j) - normalise(countsCtx[j][t]); - if (includePhraseProb) - { - normalise(countsPhrase[t]); - countsPhraseLength[t] = nPhrases[t] / countsPhraseLength[t]; - } - } - probCtx = countsCtx; - if (includePhraseProb) - { - probPhrase = countsPhrase; - probPhraseLength = countsPhraseLength; - } - - double *larray = new double[lambda.size()]; - copy(lambda.begin(), lambda.end(), larray); - print_primal_dual(larray, PHRASE_VIOLATION_WEIGHT, CONTEXT_VIOLATION_WEIGHT); - delete [] larray; - - //cout << "\tllh " << llh << endl; - //cout << "\tprior: " << prior << "\n"; - //cout << "\tcontext: " << probCtx << "\n"; - //cout << "\tphrase: " << probPhrase << "\n"; - //cout << "\tphraseLen: " << probPhraseLength << "\n"; - } - - // output class membership - for (PhraseToContextCounts::iterator pcit = concordancePhraseToContexts.begin(); - pcit != concordancePhraseToContexts.end(); ++pcit) - { - const Phrase &phrase = phrases.type(pcit->first); - for (ContextCounts::iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - Dist tagCounts = conditional_probs(phrase, context, 0); - cout << phrase << " ||| " << context << " ||| " << argmax(tagCounts) << "\n"; - } - } - - return 0; -} - -void addRandomNoise(Dist &d) -{ - for (Dist::iterator dit = d.begin(); dit != d.end(); ++dit) - *dit += rng(); -} - -void normalise(Dist &d) -{ - double z = 0; - for (Dist::iterator dit = d.begin(); dit != d.end(); ++dit) - z += *dit; - for (Dist::iterator dit = d.begin(); dit != d.end(); ++dit) - *dit /= z; -} - -void addTo(Dist &d, const Dist &e) -{ - assert(d.size() == e.size()); - for (int i = 0; i < (int) d.size(); ++i) - d[i] += e[i]; -} - -int argmax(const Dist &d) -{ - double best = d[0]; - int index = 0; - for (int i = 1; i < (int) d.size(); ++i) - { - if (d[i] > best) - { - best = d[i]; - index = i; - } - } - return index; -} - -ostream &operator<<(ostream &out, const Phrase &phrase) -{ - for (Phrase::const_iterator pit = phrase.begin(); pit != phrase.end(); ++pit) - lexicon.display(((pit == phrase.begin()) ? out : out << " "), *pit); - return out; -} - -ostream &operator<<(ostream &out, const Context &context) -{ - lexicon.display(out, get<0>(context)); - lexicon.display(out << " ", get<1>(context)); - lexicon.display(out << " <PHRASE> ", get<2>(context)); - lexicon.display(out << " ", get<3>(context)); - return out; -} - -ostream &operator<<(ostream &out, const Dist &dist) -{ - for (Dist::const_iterator dit = dist.begin(); dit != dist.end(); ++dit) - out << ((dit == dist.begin()) ? "" : " ") << *dit; - return out; -} - -ostream &operator<<(ostream &out, const ConditionalDist &dist) -{ - for (ConditionalDist::const_iterator dit = dist.begin(); dit != dist.end(); ++dit) - out << ((dit == dist.begin()) ? "" : "; ") << *dit; - return out; -} - -// FIXME: slow - just use the phrase index, context index to do the mapping -// (n.b. it's a sparse setup, not just equal to 3d array index) -int -lambda_index(const Phrase &phrase, const Context &context, int tag) -{ - return lambda_indices[phrase][context] + tag; -} - -template <typename T> -Dist -penalised_conditionals(const Phrase &phrase, const Context &context, - const T &lambda, double *normalisation) -{ - Dist d = conditional_probs(phrase, context, 0); - - double z = 0; - for (int t = 0; t < numTags; ++t) - { - d[t] *= exp(-lambda[lambda_index(phrase, context, t)]); - z += d[t]; - } - - if (normalisation) - *normalisation = z; - - for (int t = 0; t < numTags; ++t) - d[t] /= z; - - return d; -} - -Dist -conditional_probs(const Phrase &phrase, const Context &context, double *normalisation) -{ - Dist tagCounts(numTags, 0.0); - double z = 0; - for (int t = 0; t < numTags; ++t) - { - double prob = prior[t]; - prob *= (probCtx[0][t][get<0>(context)] * probCtx[1][t][get<1>(context)] * - probCtx[2][t][get<2>(context)] * probCtx[3][t][get<3>(context)]); - - if (includePhraseProb) - { - prob *= pow(1 - probPhraseLength[t], phrase.size() - 1) * probPhraseLength[t]; - for (Phrase::const_iterator pit = phrase.begin(); pit != phrase.end(); ++pit) - prob *= probPhrase[t][*pit]; - } - - tagCounts[t] = prob; - z += prob; - } - if (normalisation) - *normalisation = z; - - for (int t = 0; t < numTags; ++t) - tagCounts[t] /= z; - - return tagCounts; -} - -double -penalised_log_likelihood(int n, const double *lambda, double *grad, void *) -{ - // return log Z(lambda, theta) over the corpus - // where theta are the global parameters (prior, probCtx*, probPhrase*) - // and lambda are lagrange multipliers for the posterior sparsity constraints - // - // this is formulated as: - // f = log Z(lambda) = sum_i log ( sum_i p_theta(t_i|p_i,c_i) exp [-lambda_{t_i,p_i,c_i}] ) - // where i indexes the training examples - specifying the (p, c) pair (which may occur with count > 1) - // - // with derivative: - // f'_{tpc} = frac { - count(t,p,c) p_theta(t|p,c) exp (-lambda_{t,p,c}) } - // { sum_t' p_theta(t'|p,c) exp (-lambda_{t',p,c}) } - - //cout << "penalised_log_likelihood with lambda "; - //copy(lambda, lambda+n, ostream_iterator<double>(cout, " ")); - //cout << "\n"; - - double f = 0; - if (grad) - { - for (int i = 0; i < n; ++i) - grad[i] = 0.0; - } - - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - double z = 0; - Dist scores = penalised_conditionals(phrase, context, lambda, &z); - - f += ccit->second * log(z); - //cout << "\tphrase: " << phrase << " context: " << context << " count: " << ccit->second << " z " << z << endl; - //cout << "\t\tscores: " << scores << "\n"; - - if (grad) - { - for (int t = 0; t < numTags; ++t) - { - int i = lambda_index(phrase, context, t); // FIXME: redundant lookups - assert(grad[i] == 0.0); - grad[i] = - ccit->second * scores[t]; - } - } - } - } - - //cout << "penalised_log_likelihood returning " << f; - //if (grad) - //{ - //cout << "\ngradient: "; - //copy(grad, grad+n, ostream_iterator<double>(cout, " ")); - //} - //cout << "\n"; - - return f; -} - -typedef struct -{ - // one of p or c should be set to -1, in which case it will be marginalised out - // i.e. sum_p' lambda_{p'ct} <= threshold - // or sum_c' lambda_{pc't} <= threshold - int p, c, t, threshold; -} constraint_data; - -double -constraint_and_gradient(int n, const double *lambda, double *grad, void *data) -{ - constraint_data *d = (constraint_data *) data; - assert(d->t >= 0); - assert(d->threshold >= 0); - - //cout << "constraint_and_gradient: t " << d->t << " p " << d->p << " c " << d->c << " tau " << d->threshold << endl; - //cout << "\tlambda "; - //copy(lambda, lambda+n, ostream_iterator<double>(cout, " ")); - //cout << "\n"; - - // FIXME: it's crazy to use a dense gradient here => will only have a handful of non-zero entries - if (grad) - { - for (int i = 0; i < n; ++i) - grad[i] = 0.0; - } - - //cout << "constraint_and_gradient: " << d->p << "; " << d->c << "; " << d->t << "; " << d->threshold << endl; - - if (d->p >= 0) - { - assert(d->c < 0); - // sum_c lambda_pct <= delta [a.k.a. threshold] - // => sum_c lambda_pct - delta <= 0 - // derivative_pct = { 1, if p and t match; 0, otherwise } - - double val = -d->threshold; - - const Phrase &phrase = phrases.type(d->p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(d->p); - assert(pcit != concordancePhraseToContexts.end()); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - int i = lambda_index(phrase, context, d->t); - val += lambda[i]; - if (grad) grad[i] = 1; - } - //cout << "\treturning " << val << endl; - - return val; - } - else - { - assert(d->c >= 0); - assert(d->p < 0); - // sum_p lambda_pct <= gamma [a.k.a. threshold] - // => sum_p lambda_pct - gamma <= 0 - // derivative_pct = { 1, if c and t match; 0, otherwise } - - double val = -d->threshold; - - const Context &context = contexts.type(d->c); - ContextToPhraseCounts::iterator cpit = concordanceContextToPhrases.find(d->c); - assert(cpit != concordanceContextToPhrases.end()); - for (PhraseCounts::iterator pcit = cpit->second.begin(); - pcit != cpit->second.end(); ++pcit) - { - const Phrase &phrase = phrases.type(pcit->first); - int i = lambda_index(phrase, context, d->t); - val += lambda[i]; - if (grad) grad[i] = 1; - } - //cout << "\treturning " << val << endl; - - return val; - } -} - -void -optimise_lambda(double delta, double gamma, vector<double> &lambdav) -{ - int num_lambdas = lambdav.size(); - if (lambda_indices.empty() || lambdav.empty()) - { - lambda_indices.clear(); - lambdav.clear(); - - int i = 0; - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - lambda_indices[phrase][context] = i; - i += numTags; - } - } - num_lambdas = i; - lambdav.resize(num_lambdas); - } - //cout << "optimise_lambda: #langrange multipliers " << num_lambdas << endl; - - // FIXME: better to work with an implicit representation to save memory usage - int num_constraints = (((delta > 0) ? phrases.size() : 0) + ((gamma > 0) ? contexts.size() : 0)) * numTags; - //cout << "optimise_lambda: #constraints " << num_constraints << endl; - constraint_data *data = new constraint_data[num_constraints]; - int i = 0; - if (delta > 0) - { - for (int p = 0; p < phrases.size(); ++p) - { - for (int t = 0; t < numTags; ++t, ++i) - { - constraint_data &d = data[i]; - d.p = p; - d.c = -1; - d.t = t; - d.threshold = delta; - } - } - } - - if (gamma > 0) - { - for (int c = 0; c < contexts.size(); ++c) - { - for (int t = 0; t < numTags; ++t, ++i) - { - constraint_data &d = data[i]; - d.p = -1; - d.c = c; - d.t = t; - d.threshold = gamma; - } - } - } - assert(i == num_constraints); - - double lambda[num_lambdas]; - double lb[num_lambdas], ub[num_lambdas]; - for (i = 0; i < num_lambdas; ++i) - { - lambda[i] = lambdav[i]; // starting value - lb[i] = 0; // lower bound - if (delta <= 0) // upper bound - ub[i] = gamma; - else if (gamma <= 0) - ub[i] = delta; - else - assert(false); - } - - //print_primal_dual(lambda, delta, gamma); - - double minf; - int error_code = nlopt_minimize_constrained(NLOPT_LN_COBYLA, num_lambdas, penalised_log_likelihood, NULL, - num_constraints, constraint_and_gradient, data, sizeof(constraint_data), - lb, ub, lambda, &minf, -HUGE_VAL, 0.0, 0.0, 1e-4, NULL, 0, 0.0); - //cout << "optimise error code " << error_code << endl; - - //print_primal_dual(lambda, delta, gamma); - - delete [] data; - - if (error_code < 0) - cout << "WARNING: optimisation failed with error code: " << error_code << endl; - //else - //{ - //cout << "success; minf " << minf << endl; - //print_primal_dual(lambda, delta, gamma); - //} - - lambdav = vector<double>(&lambda[0], &lambda[0] + num_lambdas); -} - -// FIXME: inefficient - cache the scores -double -expected_violation_phrases(const double *lambda) -{ - // sum_pt max_c E_q[phi_pct] - double violation = 0; - - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - - for (int t = 0; t < numTags; ++t) - { - double best = 0; - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - Dist scores = penalised_conditionals(phrase, context, lambda, 0); - best = max(best, scores[t]); - } - violation += best; - } - } - - return violation; -} - -// FIXME: inefficient - cache the scores -double -expected_violation_contexts(const double *lambda) -{ - // sum_ct max_p E_q[phi_pct] - double violation = 0; - - for (int c = 0; c < contexts.size(); ++c) - { - const Context &context = contexts.type(c); - ContextToPhraseCounts::iterator cpit = concordanceContextToPhrases.find(c); - - for (int t = 0; t < numTags; ++t) - { - double best = 0; - for (PhraseCounts::iterator pit = cpit->second.begin(); - pit != cpit->second.end(); ++pit) - { - const Phrase &phrase = phrases.type(pit->first); - Dist scores = penalised_conditionals(phrase, context, lambda, 0); - best = max(best, scores[t]); - } - violation += best; - } - } - - return violation; -} - -// FIXME: possibly inefficient -double -primal_likelihood() // FIXME: primal evaluation needs to use lambda and calculate l1linf terms -{ - double llh = 0; - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - double z = 0; - Dist scores = conditional_probs(phrase, context, &z); - llh += ccit->second * log(z); - } - } - return llh; -} - -// FIXME: inefficient - cache the scores -double -primal_kl_divergence(const double *lambda) -{ - // return KL(q || p) = sum_y q(y) { log q(y) - log p(y | x) } - // = sum_y q(y) { log p(y | x) - lambda . phi(x, y) - log Z - log p(y | x) } - // = sum_y q(y) { - lambda . phi(x, y) } - log Z - // and q(y) factors with each edge, ditto for Z - - double feature_sum = 0, log_z = 0; - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - - double local_z = 0; - double local_f = 0; - Dist d = conditional_probs(phrase, context, 0); - for (int t = 0; t < numTags; ++t) - { - int i = lambda_index(phrase, context, t); - double s = d[t] * exp(-lambda[i]); - local_f += lambda[i] * s; - local_z += s; - } - - log_z += ccit->second * log(local_z); - feature_sum += ccit->second * (local_f / local_z); - } - } - - return -feature_sum - log_z; -} - -// FIXME: inefficient - cache the scores -double -dual(const double *lambda) -{ - // return log(Z) = - log { sum_y p(y | x) exp( - lambda . phi(x, y) } - // n.b. have flipped the sign as we're minimising - - double z = 0; - for (int p = 0; p < phrases.size(); ++p) - { - const Phrase &phrase = phrases.type(p); - PhraseToContextCounts::const_iterator pcit = concordancePhraseToContexts.find(p); - for (ContextCounts::const_iterator ccit = pcit->second.begin(); - ccit != pcit->second.end(); ++ccit) - { - const Context &context = contexts.type(ccit->first); - double lz = 0; - Dist scores = penalised_conditionals(phrase, context, lambda, &z); - z += lz * ccit->second; - } - } - return log(z); -} - -void -print_primal_dual(const double *lambda, double delta, double gamma) -{ - double likelihood = primal_likelihood(); - double kl = primal_kl_divergence(lambda); - double sum_pt = expected_violation_phrases(lambda); - double sum_ct = expected_violation_contexts(lambda); - //double d = dual(lambda); - - cout << "\tllh=" << likelihood - << " kl=" << kl - << " violations phrases=" << sum_pt - << " contexts=" << sum_ct - //<< " primal=" << (kl + delta * sum_pt + gamma * sum_ct) - //<< " dual=" << d - << " objective=" << (likelihood - kl + delta * sum_pt + gamma * sum_ct) - << endl; -} diff --git a/gi/posterior-regularisation/invert.hh b/gi/posterior-regularisation/invert.hh deleted file mode 100644 index d06356e9..00000000 --- a/gi/posterior-regularisation/invert.hh +++ /dev/null @@ -1,45 +0,0 @@ -// The following code inverts the matrix input using LU-decomposition with -// backsubstitution of unit vectors. Reference: Numerical Recipies in C, 2nd -// ed., by Press, Teukolsky, Vetterling & Flannery. -// Code written by Fredrik Orderud. -// http://www.crystalclearsoftware.com/cgi-bin/boost_wiki/wiki.pl?LU_Matrix_Inversion - -#ifndef INVERT_MATRIX_HPP -#define INVERT_MATRIX_HPP - -// REMEMBER to update "lu.hpp" header includes from boost-CVS -#include <boost/numeric/ublas/vector.hpp> -#include <boost/numeric/ublas/vector_proxy.hpp> -#include <boost/numeric/ublas/matrix.hpp> -#include <boost/numeric/ublas/triangular.hpp> -#include <boost/numeric/ublas/lu.hpp> -#include <boost/numeric/ublas/io.hpp> - -namespace ublas = boost::numeric::ublas; - -/* Matrix inversion routine. - Uses lu_factorize and lu_substitute in uBLAS to invert a matrix */ -template<class T> -bool invert_matrix(const ublas::matrix<T>& input, ublas::matrix<T>& inverse) -{ - using namespace boost::numeric::ublas; - typedef permutation_matrix<std::size_t> pmatrix; - // create a working copy of the input - matrix<T> A(input); - // create a permutation matrix for the LU-factorization - pmatrix pm(A.size1()); - - // perform LU-factorization - int res = lu_factorize(A,pm); - if( res != 0 ) return false; - - // create identity matrix of "inverse" - inverse.assign(ublas::identity_matrix<T>(A.size1())); - - // backsubstitute to get the inverse - lu_substitute(A, pm, inverse); - - return true; -} - -#endif //INVERT_MATRIX_HPP diff --git a/gi/posterior-regularisation/linesearch.py b/gi/posterior-regularisation/linesearch.py deleted file mode 100644 index 5a3f2e9c..00000000 --- a/gi/posterior-regularisation/linesearch.py +++ /dev/null @@ -1,58 +0,0 @@ -## Automatically adapted for scipy Oct 07, 2005 by convertcode.py - -from scipy.optimize import minpack2 -import numpy - -import __builtin__ -pymin = __builtin__.min - -def line_search(f, myfprime, xk, pk, gfk, old_fval, old_old_fval, - args=(), c1=1e-4, c2=0.9, amax=50): - - fc = 0 - gc = 0 - phi0 = old_fval - derphi0 = numpy.dot(gfk,pk) - alpha1 = pymin(1.0,1.01*2*(phi0-old_old_fval)/derphi0) - # trevor: added this test - alpha1 = pymin(alpha1,amax) - - if isinstance(myfprime,type(())): - eps = myfprime[1] - fprime = myfprime[0] - newargs = (f,eps) + args - gradient = False - else: - fprime = myfprime - newargs = args - gradient = True - - xtol = 1e-14 - amin = 1e-8 - isave = numpy.zeros((2,), numpy.intc) - dsave = numpy.zeros((13,), float) - task = 'START' - fval = old_fval - gval = gfk - - while 1: - stp,fval,derphi,task = minpack2.dcsrch(alpha1, phi0, derphi0, c1, c2, - xtol, task, amin, amax,isave,dsave) - #print 'minpack2.dcsrch', alpha1, phi0, derphi0, c1, c2, xtol, task, amin, amax,isave,dsave - #print 'returns', stp,fval,derphi,task - - if task[:2] == 'FG': - alpha1 = stp - fval = f(xk+stp*pk,*args) - fc += 1 - gval = fprime(xk+stp*pk,*newargs) - if gradient: gc += 1 - else: fc += len(xk) + 1 - phi0 = fval - derphi0 = numpy.dot(gval,pk) - else: - break - - if task[:5] == 'ERROR' or task[1:4] == 'WARN': - stp = None # failed - return stp, fc, gc, fval, old_fval, gval diff --git a/gi/posterior-regularisation/log_add.hh b/gi/posterior-regularisation/log_add.hh deleted file mode 100644 index e0620c5a..00000000 --- a/gi/posterior-regularisation/log_add.hh +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef log_add_hh -#define log_add_hh - -#include <limits> -#include <iostream> -#include <cassert> -#include <cmath> - -template <typename T> -struct Log -{ - static T zero() { return -std::numeric_limits<T>::infinity(); } - - static T add(T l1, T l2) - { - if (l1 == zero()) return l2; - if (l1 > l2) - return l1 + std::log(1 + exp(l2 - l1)); - else - return l2 + std::log(1 + exp(l1 - l2)); - } - - static T subtract(T l1, T l2) - { - //std::assert(l1 >= l2); - return l1 + log(1 - exp(l2 - l1)); - } -}; - -#endif diff --git a/gi/posterior-regularisation/prjava.jar b/gi/posterior-regularisation/prjava.jar deleted file mode 120000 index da8bf761..00000000 --- a/gi/posterior-regularisation/prjava.jar +++ /dev/null @@ -1 +0,0 @@ -prjava/prjava-20100708.jar
\ No newline at end of file diff --git a/gi/posterior-regularisation/prjava/Makefile b/gi/posterior-regularisation/prjava/Makefile deleted file mode 100755 index bd3bfca0..00000000 --- a/gi/posterior-regularisation/prjava/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -all: - ant dist - -check: - echo no tests - -clean: - ant clean diff --git a/gi/posterior-regularisation/prjava/build.xml b/gi/posterior-regularisation/prjava/build.xml deleted file mode 100644 index 7222b3c8..00000000 --- a/gi/posterior-regularisation/prjava/build.xml +++ /dev/null @@ -1,38 +0,0 @@ -<project name="prjava" default="dist" basedir="."> - <!-- set global properties for this build --> - <property name="src" location="src"/> - <property name="build" location="build"/> - <property name="dist" location="lib"/> - <path id="classpath"> - <pathelement location="lib/trove-2.0.2.jar"/> - <pathelement location="lib/optimization.jar"/> - <pathelement location="lib/jopt-simple-3.2.jar"/> - <pathelement location="lib/commons-math-2.1.jar"/> - </path> - - <target name="init"> - <!-- Create the time stamp --> - <tstamp/> - <!-- Create the build directory structure used by compile --> - <mkdir dir="${build}"/> - </target> - - <target name="compile" depends="init" - description="compile the source " > - <!-- Compile the java code from ${src} into ${build} --> - <javac srcdir="${src}" destdir="${build}" includeantruntime="false"> - <classpath refid="classpath"/> - </javac> - </target> - - <target name="dist" depends="compile" - description="generate the distribution" > - <jar jarfile="${dist}/prjava-${DSTAMP}.jar" basedir="${build}"/> - <symlink link="./prjava.jar" resource="${dist}/prjava-${DSTAMP}.jar" overwrite="true"/> - </target> - - <target name="clean" - description="clean up" > - <delete dir="${build}"/> - </target> -</project> diff --git a/gi/posterior-regularisation/prjava/lib/commons-math-2.1.jar b/gi/posterior-regularisation/prjava/lib/commons-math-2.1.jar Binary files differdeleted file mode 100644 index 43b4b369..00000000 --- a/gi/posterior-regularisation/prjava/lib/commons-math-2.1.jar +++ /dev/null diff --git a/gi/posterior-regularisation/prjava/lib/jopt-simple-3.2.jar b/gi/posterior-regularisation/prjava/lib/jopt-simple-3.2.jar Binary files differdeleted file mode 100644 index 56373621..00000000 --- a/gi/posterior-regularisation/prjava/lib/jopt-simple-3.2.jar +++ /dev/null diff --git a/gi/posterior-regularisation/prjava/lib/trove-2.0.2.jar b/gi/posterior-regularisation/prjava/lib/trove-2.0.2.jar Binary files differdeleted file mode 100644 index 3e59fbf3..00000000 --- a/gi/posterior-regularisation/prjava/lib/trove-2.0.2.jar +++ /dev/null diff --git a/gi/posterior-regularisation/prjava/src/arr/F.java b/gi/posterior-regularisation/prjava/src/arr/F.java deleted file mode 100644 index be0a6ed6..00000000 --- a/gi/posterior-regularisation/prjava/src/arr/F.java +++ /dev/null @@ -1,99 +0,0 @@ -package arr;
-
-import java.util.Arrays;
-import java.util.Random;
-
-public class F {
- public static Random rng = new Random();
-
- public static void randomise(double probs[])
- {
- randomise(probs, true);
- }
-
- public static void randomise(double probs[], boolean normalise)
- {
- double z = 0;
- for (int i = 0; i < probs.length; ++i)
- {
- probs[i] = 10 + rng.nextDouble();
- if (normalise)
- z += probs[i];
- }
-
- if (normalise)
- for (int i = 0; i < probs.length; ++i)
- probs[i] /= z;
- }
-
- public static void uniform(double probs[])
- {
- for (int i = 0; i < probs.length; ++i)
- probs[i] = 1.0 / probs.length;
- }
-
- public static void l1normalize(double [] a){
- double sum=0;
- for(int i=0;i<a.length;i++){
- sum+=a[i];
- }
- if(sum==0)
- Arrays.fill(a, 1.0/a.length);
- else
- {
- for(int i=0;i<a.length;i++){
- a[i]/=sum;
- }
- }
- }
-
- public static void l1normalize(double [][] a){
- double sum=0;
- for(int i=0;i<a.length;i++){
- for(int j=0;j<a[i].length;j++){
- sum+=a[i][j];
- }
- }
- if(sum==0){
- return;
- }
- for(int i=0;i<a.length;i++){
- for(int j=0;j<a[i].length;j++){
- a[i][j]/=sum;
- }
- }
- }
-
- public static double l1norm(double a[]){
- // FIXME: this isn't the l1 norm for a < 0
- double norm=0;
- for(int i=0;i<a.length;i++){
- norm += a[i];
- }
- return norm;
- }
-
- public static double l2norm(double a[]){
- double norm=0;
- for(int i=0;i<a.length;i++){
- norm += a[i]*a[i];
- }
- return Math.sqrt(norm);
- }
-
- public static int argmax(double probs[])
- {
- double m = Double.NEGATIVE_INFINITY;
- int mi = -1;
- for (int i = 0; i < probs.length; ++i)
- {
- if (probs[i] > m)
- {
- m = probs[i];
- mi = i;
- }
- }
- return mi;
- }
-
-}
diff --git a/gi/posterior-regularisation/prjava/src/data/Corpus.java b/gi/posterior-regularisation/prjava/src/data/Corpus.java deleted file mode 100644 index 425ede11..00000000 --- a/gi/posterior-regularisation/prjava/src/data/Corpus.java +++ /dev/null @@ -1,233 +0,0 @@ -package data;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Scanner;
-
-public class Corpus {
-
- public static final String alphaFilename="../posdata/corpus.alphabet";
- public static final String tagalphaFilename="../posdata/corpus.tag.alphabet";
-
-// public static final String START_SYM="<s>";
- public static final String END_SYM="<e>";
- public static final String NUM_TOK="<NUM>";
-
- public static final String UNK_TOK="<unk>";
-
- private ArrayList<String[]>sent;
- private ArrayList<int[]>data;
-
- public ArrayList<String[]>tag;
- public ArrayList<int[]>tagData;
-
- public static boolean convertNumTok=true;
-
- private HashMap<String,Integer>freq;
- public HashMap<String,Integer>vocab;
-
- public HashMap<String,Integer>tagVocab;
- private int tagV;
-
- private int V;
-
- public static void main(String[] args) {
- Corpus c=new Corpus("../posdata/en_test.conll");
- System.out.println(
- Arrays.toString(c.get(0))
- );
- System.out.println(
- Arrays.toString(c.getInt(0))
- );
-
- System.out.println(
- Arrays.toString(c.get(1))
- );
- System.out.println(
- Arrays.toString(c.getInt(1))
- );
- }
-
- public Corpus(String filename,HashMap<String,Integer>dict){
- V=0;
- tagV=0;
- freq=new HashMap<String,Integer>();
- tagVocab=new HashMap<String,Integer>();
- vocab=dict;
-
- sent=new ArrayList<String[]>();
- tag=new ArrayList<String[]>();
-
- Scanner sc=io.FileUtil.openInFile(filename);
- ArrayList<String>s=new ArrayList<String>();
- // s.add(START_SYM);
- while(sc.hasNextLine()){
- String line=sc.nextLine();
- String toks[]=line.split("\t");
- if(toks.length<2){
- s.add(END_SYM);
- sent.add(s.toArray(new String[0]));
- s=new ArrayList<String>();
- // s.add(START_SYM);
- continue;
- }
- String tok=toks[1].toLowerCase();
- s.add(tok);
- }
- sc.close();
-
- buildData();
- }
-
- public Corpus(String filename){
- V=0;
- freq=new HashMap<String,Integer>();
- vocab=new HashMap<String,Integer>();
- tagVocab=new HashMap<String,Integer>();
-
- sent=new ArrayList<String[]>();
- tag=new ArrayList<String[]>();
-
- System.out.println("Reading:"+filename);
-
- Scanner sc=io.FileUtil.openInFile(filename);
- ArrayList<String>s=new ArrayList<String>();
- ArrayList<String>tags=new ArrayList<String>();
- //s.add(START_SYM);
- while(sc.hasNextLine()){
- String line=sc.nextLine();
- String toks[]=line.split("\t");
- if(toks.length<2){
- s.add(END_SYM);
- tags.add(END_SYM);
- if(s.size()>2){
- sent.add(s.toArray(new String[0]));
- tag.add(tags.toArray(new String [0]));
- }
- s=new ArrayList<String>();
- tags=new ArrayList<String>();
- // s.add(START_SYM);
- continue;
- }
-
- String tok=toks[1].toLowerCase();
- if(convertNumTok && tok.matches(".*\\d.*")){
- tok=NUM_TOK;
- }
- s.add(tok);
-
- if(toks.length>3){
- tok=toks[3].toLowerCase();
- }else{
- tok="_";
- }
- tags.add(tok);
-
- }
- sc.close();
-
- for(int i=0;i<sent.size();i++){
- String[]toks=sent.get(i);
- for(int j=0;j<toks.length;j++){
- addVocab(toks[j]);
- addTag(tag.get(i)[j]);
- }
- }
-
- buildVocab();
- buildData();
- System.out.println(data.size()+"sentences, "+vocab.keySet().size()+" word types");
- }
-
- public String[] get(int idx){
- return sent.get(idx);
- }
-
- private void addVocab(String s){
- Integer integer=freq.get(s);
- if(integer==null){
- integer=0;
- }
- freq.put(s, integer+1);
- }
-
- public int tokIdx(String tok){
- Integer integer=vocab.get(tok);
- if(integer==null){
- return V;
- }
- return integer;
- }
-
- public int tagIdx(String tok){
- Integer integer=tagVocab.get(tok);
- if(integer==null){
- return tagV;
- }
- return integer;
- }
-
- private void buildData(){
- data=new ArrayList<int[]>();
- for(int i=0;i<sent.size();i++){
- String s[]=sent.get(i);
- data.add(new int [s.length]);
- for(int j=0;j<s.length;j++){
- data.get(i)[j]=tokIdx(s[j]);
- }
- }
-
- tagData=new ArrayList<int[]>();
- for(int i=0;i<tag.size();i++){
- String s[]=tag.get(i);
- tagData.add(new int [s.length]);
- for(int j=0;j<s.length;j++){
- tagData.get(i)[j]=tagIdx(s[j]);
- }
- }
- sent=null;
- tag=null;
- System.gc();
- }
-
- public int [] getInt(int idx){
- return data.get(idx);
- }
-
- /**
- *
- * @return size of vocabulary
- */
- public int getVocabSize(){
- return V;
- }
-
- public int [][]getAllData(){
- return data.toArray(new int [0][]);
- }
-
- public int [][]getTagData(){
- return tagData.toArray(new int [0][]);
- }
-
- private void buildVocab(){
- for (String key:freq.keySet()){
- if(freq.get(key)>2){
- vocab.put(key, V);
- V++;
- }
- }
- io.SerializedObjects.writeSerializedObject(vocab, alphaFilename);
- io.SerializedObjects.writeSerializedObject(tagVocab,tagalphaFilename);
- }
-
- private void addTag(String tag){
- Integer i=tagVocab.get(tag);
- if(i==null){
- tagVocab.put(tag, tagV);
- tagV++;
- }
- }
-
-}
diff --git a/gi/posterior-regularisation/prjava/src/hmm/HMM.java b/gi/posterior-regularisation/prjava/src/hmm/HMM.java deleted file mode 100644 index 17a4679f..00000000 --- a/gi/posterior-regularisation/prjava/src/hmm/HMM.java +++ /dev/null @@ -1,579 +0,0 @@ -package hmm;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Scanner;
-
-public class HMM {
-
-
- //trans[i][j]=prob of going FROM i to j
- double [][]trans;
- double [][]emit;
- double []pi;
- int [][]data;
- int [][]tagdata;
-
- double logtrans[][];
-
- public HMMObjective o;
-
- public static void main(String[] args) {
-
- }
-
- public HMM(int n_state,int n_emit,int [][]data){
- trans=new double [n_state][n_state];
- emit=new double[n_state][n_emit];
- pi=new double [n_state];
- System.out.println(" random initial parameters");
- fillRand(trans);
- fillRand(emit);
- fillRand(pi);
-
- this.data=data;
-
- }
-
- private void fillRand(double [][] a){
- for(int i=0;i<a.length;i++){
- for(int j=0;j<a[i].length;j++){
- a[i][j]=Math.random();
- }
- l1normalize(a[i]);
- }
- }
- private void fillRand(double []a){
- for(int i=0;i<a.length;i++){
- a[i]=Math.random();
- }
- l1normalize(a);
- }
-
- private double loglikely=0;
-
- public void EM(){
- double trans_exp_cnt[][]=new double [trans.length][trans.length];
- double emit_exp_cnt[][]=new double[trans.length][emit[0].length];
- double start_exp_cnt[]=new double[trans.length];
- loglikely=0;
-
- //E
- for(int i=0;i<data.length;i++){
-
- double [][][] post=forwardBackward(data[i]);
- incrementExpCnt(post, data[i],
- trans_exp_cnt,
- emit_exp_cnt,
- start_exp_cnt);
-
-
- if(i%100==0){
- System.out.print(".");
- }
- if(i%1000==0){
- System.out.println(i);
- }
-
- }
- System.out.println("Log likelihood: "+loglikely);
-
- //M
- addOneSmooth(emit_exp_cnt);
- for(int i=0;i<trans.length;i++){
-
- //transition probs
- double sum=0;
- for(int j=0;j<trans.length;j++){
- sum+=trans_exp_cnt[i][j];
- }
- //avoid NAN
- if(sum==0){
- sum=1;
- }
- for(int j=0;j<trans[i].length;j++){
- trans[i][j]=trans_exp_cnt[i][j]/sum;
- }
-
- //emission probs
-
- sum=0;
- for(int j=0;j<emit[i].length;j++){
- sum+=emit_exp_cnt[i][j];
- }
- //avoid NAN
- if(sum==0){
- sum=1;
- }
- for(int j=0;j<emit[i].length;j++){
- emit[i][j]=emit_exp_cnt[i][j]/sum;
- }
-
-
- //initial probs
- for(int j=0;j<pi.length;j++){
- pi[j]=start_exp_cnt[j];
- }
- l1normalize(pi);
- }
- }
-
- private double [][][]forwardBackward(int [] seq){
- double a[][]=new double [seq.length][trans.length];
- double b[][]=new double [seq.length][trans.length];
-
- int len=seq.length;
- //initialize the first step
- for(int i=0;i<trans.length;i++){
- a[0][i]=emit[i][seq[0]]*pi[i];
- b[len-1][i]=1;
- }
-
- //log of denominator for likelyhood
- double c=Math.log(l1norm(a[0]));
-
- l1normalize(a[0]);
- l1normalize(b[len-1]);
-
-
-
- //forward
- for(int n=1;n<len;n++){
- for(int i=0;i<trans.length;i++){
- for(int j=0;j<trans.length;j++){
- a[n][i]+=trans[j][i]*a[n-1][j];
- }
- a[n][i]*=emit[i][seq[n]];
- }
- c+=Math.log(l1norm(a[n]));
- l1normalize(a[n]);
- }
-
- loglikely+=c;
-
- //backward
- for(int n=len-2;n>=0;n--){
- for(int i=0;i<trans.length;i++){
- for(int j=0;j<trans.length;j++){
- b[n][i]+=trans[i][j]*b[n+1][j]*emit[j][seq[n+1]];
- }
- }
- l1normalize(b[n]);
- }
-
-
- //expected transition
- double p[][][]=new double [seq.length][trans.length][trans.length];
- for(int n=0;n<len-1;n++){
- for(int i=0;i<trans.length;i++){
- for(int j=0;j<trans.length;j++){
- p[n][i][j]=a[n][i]*trans[i][j]*emit[j][seq[n+1]]*b[n+1][j];
-
- }
- }
-
- l1normalize(p[n]);
- }
- return p;
- }
-
- private void incrementExpCnt(
- double post[][][],int [] seq,
- double trans_exp_cnt[][],
- double emit_exp_cnt[][],
- double start_exp_cnt[])
- {
-
- for(int n=0;n<post.length;n++){
- for(int i=0;i<trans.length;i++){
- double py=0;
- for(int j=0;j<trans.length;j++){
- py+=post[n][i][j];
- trans_exp_cnt[i][j]+=post[n][i][j];
- }
-
- emit_exp_cnt[i][seq[n]]+=py;
-
- }
- }
-
- //the first state
- for(int i=0;i<trans.length;i++){
- double py=0;
- for(int j=0;j<trans.length;j++){
- py+=post[0][i][j];
- }
- start_exp_cnt[i]+=py;
- }
-
-
- //the last state
- int len=post.length;
- for(int i=0;i<trans.length;i++){
- double py=0;
- for(int j=0;j<trans.length;j++){
- py+=post[len-2][j][i];
- }
- emit_exp_cnt[i][seq[len-1]]+=py;
- }
- }
-
- public void l1normalize(double [] a){
- double sum=0;
- for(int i=0;i<a.length;i++){
- sum+=a[i];
- }
- if(sum==0){
- return ;
- }
- for(int i=0;i<a.length;i++){
- a[i]/=sum;
- }
- }
-
- public void l1normalize(double [][] a){
- double sum=0;
- for(int i=0;i<a.length;i++){
- for(int j=0;j<a[i].length;j++){
- sum+=a[i][j];
- }
- }
- if(sum==0){
- return;
- }
- for(int i=0;i<a.length;i++){
- for(int j=0;j<a[i].length;j++){
- a[i][j]/=sum;
- }
- }
- }
-
- public void writeModel(String modelFilename) throws FileNotFoundException, IOException{
- PrintStream ps=io.FileUtil.printstream(new File(modelFilename));
- ps.println(trans.length);
- ps.println("Initial Probabilities:");
- for(int i=0;i<pi.length;i++){
- ps.print(pi[i]+"\t");
- }
- ps.println();
- ps.println("Transition Probabilities:");
- for(int i=0;i<trans.length;i++){
- for(int j=0;j<trans[i].length;j++){
- ps.print(trans[i][j]+"\t");
- }
- ps.println();
- }
- ps.println("Emission Probabilities:");
- ps.println(emit[0].length);
- for(int i=0;i<trans.length;i++){
- for(int j=0;j<emit[i].length;j++){
- ps.println(emit[i][j]);
- }
- ps.println();
- }
- ps.close();
- }
-
- public HMM(){
-
- }
-
- public void readModel(String modelFilename){
- Scanner sc=io.FileUtil.openInFile(modelFilename);
-
- int n_state=sc.nextInt();
- sc.nextLine();
- sc.nextLine();
- pi=new double [n_state];
- for(int i=0;i<n_state;i++){
- pi[i]=sc.nextDouble();
- }
- sc.nextLine();
- sc.nextLine();
- trans=new double[n_state][n_state];
- for(int i=0;i<trans.length;i++){
- for(int j=0;j<trans[i].length;j++){
- trans[i][j]=sc.nextDouble();
- }
- }
- sc.nextLine();
- sc.nextLine();
-
- int n_obs=sc.nextInt();
- emit=new double[n_state][n_obs];
- for(int i=0;i<trans.length;i++){
- for(int j=0;j<emit[i].length;j++){
- emit[i][j]=sc.nextDouble();
- }
- }
- sc.close();
- }
-
- public int []viterbi(int [] seq){
- double [][]p=new double [seq.length][trans.length];
- int backp[][]=new int [seq.length][trans.length];
-
- for(int i=0;i<trans.length;i++){
- p[0][i]=Math.log(emit[i][seq[0]]*pi[i]);
- }
-
- double a[][]=logtrans;
- if(logtrans==null){
- a=new double [trans.length][trans.length];
- for(int i=0;i<trans.length;i++){
- for(int j=0;j<trans.length;j++){
- a[i][j]=Math.log(trans[i][j]);
- }
- }
- logtrans=a;
- }
-
- double maxprob=0;
- for(int n=1;n<seq.length;n++){
- for(int i=0;i<trans.length;i++){
- maxprob=p[n-1][0]+a[0][i];
- backp[n][i]=0;
- for(int j=1;j<trans.length;j++){
- double prob=p[n-1][j]+a[j][i];
- if(maxprob<prob){
- backp[n][i]=j;
- maxprob=prob;
- }
- }
- p[n][i]=maxprob+Math.log(emit[i][seq[n]]);
- }
- }
-
- maxprob=p[seq.length-1][0];
- int maxIdx=0;
- for(int i=1;i<trans.length;i++){
- if(p[seq.length-1][i]>maxprob){
- maxprob=p[seq.length-1][i];
- maxIdx=i;
- }
- }
- int ans[]=new int [seq.length];
- ans[seq.length-1]=maxIdx;
- for(int i=seq.length-2;i>=0;i--){
- ans[i]=backp[i+1][ans[i+1]];
- }
- return ans;
- }
-
- public double l1norm(double a[]){
- double norm=0;
- for(int i=0;i<a.length;i++){
- norm += a[i];
- }
- return norm;
- }
-
- public double [][]getEmitProb(){
- return emit;
- }
-
- public int [] sample(int terminalSym){
- ArrayList<Integer > s=new ArrayList<Integer>();
- int state=sample(pi);
- int sym=sample(emit[state]);
- while(sym!=terminalSym){
- s.add(sym);
- state=sample(trans[state]);
- sym=sample(emit[state]);
- }
-
- int ans[]=new int [s.size()];
- for(int i=0;i<ans.length;i++){
- ans[i]=s.get(i);
- }
- return ans;
- }
-
- public int sample(double p[]){
- double r=Math.random();
- double sum=0;
- for(int i=0;i<p.length;i++){
- sum+=p[i];
- if(sum>=r){
- return i;
- }
- }
- return p.length-1;
- }
-
- public void train(int tagdata[][]){
- double trans_exp_cnt[][]=new double [trans.length][trans.length];
- double emit_exp_cnt[][]=new double[trans.length][emit[0].length];
- double start_exp_cnt[]=new double[trans.length];
-
- for(int i=0;i<tagdata.length;i++){
- start_exp_cnt[tagdata[i][0]]++;
-
- for(int j=0;j<tagdata[i].length;j++){
- if(j+1<tagdata[i].length){
- trans_exp_cnt[ tagdata[i][j] ] [ tagdata[i][j+1] ]++;
- }
- emit_exp_cnt[tagdata[i][j]][data[i][j]]++;
- }
-
- }
-
- //M
- addOneSmooth(emit_exp_cnt);
- for(int i=0;i<trans.length;i++){
-
- //transition probs
- double sum=0;
- for(int j=0;j<trans.length;j++){
- sum+=trans_exp_cnt[i][j];
- }
- if(sum==0){
- sum=1;
- }
- for(int j=0;j<trans[i].length;j++){
- trans[i][j]=trans_exp_cnt[i][j]/sum;
- }
-
- //emission probs
-
- sum=0;
- for(int j=0;j<emit[i].length;j++){
- sum+=emit_exp_cnt[i][j];
- }
- if(sum==0){
- sum=1;
- }
- for(int j=0;j<emit[i].length;j++){
- emit[i][j]=emit_exp_cnt[i][j]/sum;
- }
-
-
- //initial probs
- for(int j=0;j<pi.length;j++){
- pi[j]=start_exp_cnt[j];
- }
- l1normalize(pi);
- }
- }
-
- private void addOneSmooth(double a[][]){
- for(int i=0;i<a.length;i++){
- for(int j=0;j<a[i].length;j++){
- a[i][j]+=0.01;
- }
- //l1normalize(a[i]);
- }
- }
-
- public void PREM(){
-
- o.optimizeWithProjectedGradientDescent();
-
- double trans_exp_cnt[][]=new double [trans.length][trans.length];
- double emit_exp_cnt[][]=new double[trans.length][emit[0].length];
- double start_exp_cnt[]=new double[trans.length];
-
- o.loglikelihood=0;
- //E
- for(int sentNum=0;sentNum<data.length;sentNum++){
-
- double [][][] post=o.forwardBackward(sentNum);
- incrementExpCnt(post, data[sentNum],
- trans_exp_cnt,
- emit_exp_cnt,
- start_exp_cnt);
-
-
- if(sentNum%100==0){
- System.out.print(".");
- }
- if(sentNum%1000==0){
- System.out.println(sentNum);
- }
-
- }
-
- System.out.println("Log likelihood: "+o.getValue());
-
- //M
- addOneSmooth(emit_exp_cnt);
- for(int i=0;i<trans.length;i++){
-
- //transition probs
- double sum=0;
- for(int j=0;j<trans.length;j++){
- sum+=trans_exp_cnt[i][j];
- }
- //avoid NAN
- if(sum==0){
- sum=1;
- }
- for(int j=0;j<trans[i].length;j++){
- trans[i][j]=trans_exp_cnt[i][j]/sum;
- }
-
- //emission probs
-
- sum=0;
- for(int j=0;j<emit[i].length;j++){
- sum+=emit_exp_cnt[i][j];
- }
- //avoid NAN
- if(sum==0){
- sum=1;
- }
- for(int j=0;j<emit[i].length;j++){
- emit[i][j]=emit_exp_cnt[i][j]/sum;
- }
-
-
- //initial probs
- for(int j=0;j<pi.length;j++){
- pi[j]=start_exp_cnt[j];
- }
- l1normalize(pi);
- }
-
- }
-
- public void computeMaxwt(double[][]maxwt, int[][] d){
-
- for(int sentNum=0;sentNum<d.length;sentNum++){
- double post[][][]=forwardBackward(d[sentNum]);
-
- for(int n=0;n<post.length;n++){
- for(int i=0;i<trans.length;i++){
- double py=0;
- for(int j=0;j<trans.length;j++){
- py+=post[n][i][j];
- }
-
- if(py>maxwt[i][d[sentNum][n]]){
- maxwt[i][d[sentNum][n]]=py;
- }
-
- }
- }
-
- //the last state
- int len=post.length;
- for(int i=0;i<trans.length;i++){
- double py=0;
- for(int j=0;j<trans.length;j++){
- py+=post[len-2][j][i];
- }
-
- if(py>maxwt[i][d[sentNum][len-1]]){
- maxwt[i][d[sentNum][len-1]]=py;
- }
-
- }
-
- }
-
- }
-
-}//end of class
diff --git a/gi/posterior-regularisation/prjava/src/hmm/HMMObjective.java b/gi/posterior-regularisation/prjava/src/hmm/HMMObjective.java deleted file mode 100644 index 70b6c966..00000000 --- a/gi/posterior-regularisation/prjava/src/hmm/HMMObjective.java +++ /dev/null @@ -1,351 +0,0 @@ -package hmm;
-
-import gnu.trove.TIntArrayList;
-import optimization.gradientBasedMethods.ProjectedGradientDescent;
-import optimization.gradientBasedMethods.ProjectedObjective;
-import optimization.gradientBasedMethods.stats.OptimizerStats;
-import optimization.linesearch.ArmijoLineSearchMinimizationAlongProjectionArc;
-import optimization.linesearch.InterpolationPickFirstStep;
-import optimization.linesearch.LineSearchMethod;
-import optimization.projections.SimplexProjection;
-import optimization.stopCriteria.CompositeStopingCriteria;
-import optimization.stopCriteria.ProjectedGradientL2Norm;
-import optimization.stopCriteria.StopingCriteria;
-import optimization.stopCriteria.ValueDifference;
-
-public class HMMObjective extends ProjectedObjective{
-
-
- private static final double GRAD_DIFF = 3;
- public static double INIT_STEP_SIZE=10;
- public static double VAL_DIFF=1000;
-
- private HMM hmm;
- double[] newPoint ;
-
- //posterior[sent num][tok num][tag]=index into lambda
- private int posteriorMap[][][];
- //projection[word][tag].get(occurence)=index into lambda
- private TIntArrayList projectionMap[][];
-
- //Size of the simplex
- public double scale=10;
- private SimplexProjection projection;
-
- private int wordFreq[];
- private static int MIN_FREQ=10;
- private int numWordsToProject=0;
-
- private int n_param;
-
- public double loglikelihood;
-
- public HMMObjective(HMM h){
- hmm=h;
-
- countWords();
- buildMap();
-
- gradient=new double [n_param];
- projection = new SimplexProjection(scale);
- newPoint = new double[n_param];
- setInitialParameters(new double[n_param]);
-
- }
-
- /**@brief counts word frequency in the corpus
- *
- */
- private void countWords(){
- wordFreq=new int [hmm.emit[0].length];
- for(int i=0;i<hmm.data.length;i++){
- for(int j=0;j<hmm.data[i].length;j++){
- wordFreq[hmm.data[i][j]]++;
- }
- }
- }
-
- /**@brief build posterior and projection indices
- *
- */
- private void buildMap(){
- //number of sentences hidden states and words
- int n_states=hmm.trans.length;
- int n_words=hmm.emit[0].length;
- int n_sents=hmm.data.length;
-
- n_param=0;
- posteriorMap=new int[n_sents][][];
- projectionMap=new TIntArrayList[n_words][];
- for(int sentNum=0;sentNum<n_sents;sentNum++){
- int [] data=hmm.data[sentNum];
- posteriorMap[sentNum]=new int[data.length][n_states];
- numWordsToProject=0;
- for(int i=0;i<data.length;i++){
- int word=data[i];
- for(int state=0;state<n_states;state++){
- if(wordFreq[word]>MIN_FREQ){
- if(projectionMap[word]==null){
- projectionMap[word]=new TIntArrayList[n_states];
- }
- // if(posteriorMap[sentNum][i]==null){
- // posteriorMap[sentNum][i]=new int[n_states];
- // }
-
- posteriorMap[sentNum][i][state]=n_param;
- if(projectionMap[word][state]==null){
- projectionMap[word][state]=new TIntArrayList();
- numWordsToProject++;
- }
- projectionMap[word][state].add(n_param);
- n_param++;
- }
- else{
- posteriorMap[sentNum][i][state]=-1;
- }
- }
- }
- }
- }
-
- @Override
- public double[] projectPoint(double[] point) {
- // TODO Auto-generated method stub
- for(int i=0;i<projectionMap.length;i++){
-
- if(projectionMap[i]==null){
- //this word is not constrained
- continue;
- }
-
- for(int j=0;j<projectionMap[i].length;j++){
- TIntArrayList instances=projectionMap[i][j];
- double[] toProject = new double[instances.size()];
-
- for (int k = 0; k < toProject.length; k++) {
- // System.out.print(instances.get(k) + " ");
- toProject[k] = point[instances.get(k)];
- }
-
- projection.project(toProject);
- for (int k = 0; k < toProject.length; k++) {
- newPoint[instances.get(k)]=toProject[k];
- }
- }
- }
- return newPoint;
- }
-
- @Override
- public double[] getGradient() {
- // TODO Auto-generated method stub
- gradientCalls++;
- return gradient;
- }
-
- @Override
- public double getValue() {
- // TODO Auto-generated method stub
- functionCalls++;
- return loglikelihood;
- }
-
-
- @Override
- public String toString() {
- // TODO Auto-generated method stub
- StringBuffer sb = new StringBuffer();
- for (int i = 0; i < parameters.length; i++) {
- sb.append(parameters[i]+" ");
- if(i%100==0){
- sb.append("\n");
- }
- }
- sb.append("\n");
- /*
- for (int i = 0; i < gradient.length; i++) {
- sb.append(gradient[i]+" ");
- if(i%100==0){
- sb.append("\n");
- }
- }
- sb.append("\n");
- */
- return sb.toString();
- }
-
-
- /**
- * @param seq
- * @return posterior probability of each transition
- */
- public double [][][]forwardBackward(int sentNum){
- int [] seq=hmm.data[sentNum];
- int n_states=hmm.trans.length;
- double a[][]=new double [seq.length][n_states];
- double b[][]=new double [seq.length][n_states];
-
- int len=seq.length;
-
- boolean constrained=
- (projectionMap[seq[0]]!=null);
-
- //initialize the first step
- for(int i=0;i<n_states;i++){
- a[0][i]=hmm.emit[i][seq[0]]*hmm.pi[i];
- if(constrained){
- a[0][i]*=
- Math.exp(- parameters[ posteriorMap[sentNum][0][i] ] );
- }
- b[len-1][i]=1;
- }
-
- loglikelihood+=Math.log(hmm.l1norm(a[0]));
- hmm.l1normalize(a[0]);
- hmm.l1normalize(b[len-1]);
-
- //forward
- for(int n=1;n<len;n++){
-
- constrained=
- (projectionMap[seq[n]]!=null);
-
- for(int i=0;i<n_states;i++){
- for(int j=0;j<n_states;j++){
- a[n][i]+=hmm.trans[j][i]*a[n-1][j];
- }
- a[n][i]*=hmm.emit[i][seq[n]];
-
- if(constrained){
- a[n][i]*=
- Math.exp(- parameters[ posteriorMap[sentNum][n][i] ] );
- }
-
- }
- loglikelihood+=Math.log(hmm.l1norm(a[n]));
- hmm.l1normalize(a[n]);
- }
-
- //temp variable for e^{-\lambda}
- double factor=1;
- //backward
- for(int n=len-2;n>=0;n--){
-
- constrained=
- (projectionMap[seq[n+1]]!=null);
-
- for(int i=0;i<n_states;i++){
- for(int j=0;j<n_states;j++){
-
- if(constrained){
- factor=
- Math.exp(- parameters[ posteriorMap[sentNum][n+1][j] ] );
- }else{
- factor=1;
- }
-
- b[n][i]+=hmm.trans[i][j]*b[n+1][j]*hmm.emit[j][seq[n+1]]*factor;
-
- }
- }
- hmm.l1normalize(b[n]);
- }
-
- //expected transition
- double p[][][]=new double [seq.length][n_states][n_states];
- for(int n=0;n<len-1;n++){
-
- constrained=
- (projectionMap[seq[n+1]]!=null);
-
- for(int i=0;i<n_states;i++){
- for(int j=0;j<n_states;j++){
-
- if(constrained){
- factor=
- Math.exp(- parameters[ posteriorMap[sentNum][n+1][j] ] );
- }else{
- factor=1;
- }
-
- p[n][i][j]=a[n][i]*hmm.trans[i][j]*
- hmm.emit[j][seq[n+1]]*b[n+1][j]*factor;
-
- }
- }
-
- hmm.l1normalize(p[n]);
- }
- return p;
- }
-
- public void optimizeWithProjectedGradientDescent(){
- LineSearchMethod ls =
- new ArmijoLineSearchMinimizationAlongProjectionArc
- (new InterpolationPickFirstStep(INIT_STEP_SIZE));
-
- OptimizerStats stats = new OptimizerStats();
-
-
- ProjectedGradientDescent optimizer = new ProjectedGradientDescent(ls);
- StopingCriteria stopGrad = new ProjectedGradientL2Norm(GRAD_DIFF);
- StopingCriteria stopValue = new ValueDifference(VAL_DIFF);
- CompositeStopingCriteria compositeStop = new CompositeStopingCriteria();
- compositeStop.add(stopGrad);
- compositeStop.add(stopValue);
-
- optimizer.setMaxIterations(10);
- updateFunction();
- boolean succed = optimizer.optimize(this,stats,compositeStop);
- System.out.println("Ended optimzation Projected Gradient Descent\n" + stats.prettyPrint(1));
- if(succed){
- System.out.println("Ended optimization in " + optimizer.getCurrentIteration());
- }else{
- System.out.println("Failed to optimize");
- }
- }
-
- @Override
- public void setParameters(double[] params) {
- super.setParameters(params);
- updateFunction();
- }
-
- private void updateFunction(){
-
- updateCalls++;
- loglikelihood=0;
-
- for(int sentNum=0;sentNum<hmm.data.length;sentNum++){
- double [][][]p=forwardBackward(sentNum);
-
- for(int n=0;n<p.length-1;n++){
- for(int i=0;i<p[n].length;i++){
- if(projectionMap[hmm.data[sentNum][n]]!=null){
- double posterior=0;
- for(int j=0;j<p[n][i].length;j++){
- posterior+=p[n][i][j];
- }
- gradient[posteriorMap[sentNum][n][i]]=-posterior;
- }
- }
- }
-
- //the last state
- int n=p.length-2;
- for(int i=0;i<p[n].length;i++){
- if(projectionMap[hmm.data[sentNum][n+1]]!=null){
-
- double posterior=0;
- for(int j=0;j<p[n].length;j++){
- posterior+=p[n][j][i];
- }
- gradient[posteriorMap[sentNum][n+1][i]]=-posterior;
-
- }
- }
- }
-
- }
-
-}
diff --git a/gi/posterior-regularisation/prjava/src/hmm/POS.java b/gi/posterior-regularisation/prjava/src/hmm/POS.java deleted file mode 100644 index bdcbc683..00000000 --- a/gi/posterior-regularisation/prjava/src/hmm/POS.java +++ /dev/null @@ -1,120 +0,0 @@ -package hmm;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.HashMap;
-
-import data.Corpus;
-
-public class POS {
-
- //public String trainFilename="../posdata/en_train.conll";
- public static String trainFilename="../posdata/small_train.txt";
-// public static String trainFilename="../posdata/en_test.conll";
-// public static String trainFilename="../posdata/trial1.txt";
-
- public static String testFilename="../posdata/en_test.conll";
- //public static String testFilename="../posdata/trial1.txt";
-
- public static String predFilename="../posdata/en_test.predict.conll";
- public static String modelFilename="../posdata/posModel.out";
- public static final int ITER=20;
- public static final int N_STATE=30;
-
- public static void main(String[] args) {
- //POS p=new POS();
- //POS p=new POS(true);
- try {
- PRPOS();
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
-
- public POS() throws FileNotFoundException, IOException{
- Corpus c= new Corpus(trainFilename);
- //size of vocabulary +1 for unknown tokens
- HMM hmm =new HMM(N_STATE, c.getVocabSize()+1,c.getAllData());
- for(int i=0;i<ITER;i++){
- System.out.println("Iter"+i);
- hmm.EM();
- if((i+1)%10==0){
- hmm.writeModel(modelFilename+i);
- }
- }
-
- hmm.writeModel(modelFilename);
-
- Corpus test=new Corpus(testFilename,c.vocab);
-
- PrintStream ps= io.FileUtil.printstream(new File(predFilename));
-
- int [][]data=test.getAllData();
- for(int i=0;i<data.length;i++){
- int []tag=hmm.viterbi(data[i]);
- String sent[]=test.get(i);
- for(int j=0;j<data[i].length;j++){
- ps.println(sent[j]+"\t"+tag[j]);
- }
- ps.println();
- }
- ps.close();
- }
-
- //POS induction with L1/Linf constraints
- public static void PRPOS() throws FileNotFoundException, IOException{
- Corpus c= new Corpus(trainFilename);
- //size of vocabulary +1 for unknown tokens
- HMM hmm =new HMM(N_STATE, c.getVocabSize()+1,c.getAllData());
- hmm.o=new HMMObjective(hmm);
- for(int i=0;i<ITER;i++){
- System.out.println("Iter: "+i);
- hmm.PREM();
- if((i+1)%10==0){
- hmm.writeModel(modelFilename+i);
- }
- }
-
- hmm.writeModel(modelFilename);
- }
-
-
- public POS(boolean supervised) throws FileNotFoundException, IOException{
- Corpus c= new Corpus(trainFilename);
- //size of vocabulary +1 for unknown tokens
- HMM hmm =new HMM(c.tagVocab.size() , c.getVocabSize()+1,c.getAllData());
- hmm.train(c.getTagData());
-
- hmm.writeModel(modelFilename);
-
- Corpus test=new Corpus(testFilename,c.vocab);
-
- HashMap<String, Integer>tagVocab=
- (HashMap<String, Integer>) io.SerializedObjects.readSerializedObject(Corpus.tagalphaFilename);
- String [] tagdict=new String [tagVocab.size()+1];
- for(String key:tagVocab.keySet()){
- tagdict[tagVocab.get(key)]=key;
- }
- tagdict[tagdict.length-1]=Corpus.UNK_TOK;
-
- System.out.println(c.vocab.get("<e>"));
-
- PrintStream ps= io.FileUtil.printstream(new File(predFilename));
-
- int [][]data=test.getAllData();
- for(int i=0;i<data.length;i++){
- int []tag=hmm.viterbi(data[i]);
- String sent[]=test.get(i);
- for(int j=0;j<data[i].length;j++){
- ps.println(sent[j]+"\t"+tagdict[tag[j]]);
- }
- ps.println();
- }
- ps.close();
- }
-}
diff --git a/gi/posterior-regularisation/prjava/src/io/FileUtil.java b/gi/posterior-regularisation/prjava/src/io/FileUtil.java deleted file mode 100644 index 6720d087..00000000 --- a/gi/posterior-regularisation/prjava/src/io/FileUtil.java +++ /dev/null @@ -1,48 +0,0 @@ -package io;
-import java.util.*;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-import java.io.*;
-public class FileUtil
-{
- public static BufferedReader reader(File file) throws FileNotFoundException, IOException
- {
- if (file.getName().endsWith(".gz"))
- return new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(file)), "UTF8"));
- else
- return new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF8"));
- }
-
- public static PrintStream printstream(File file) throws FileNotFoundException, IOException
- {
- if (file.getName().endsWith(".gz"))
- return new PrintStream(new GZIPOutputStream(new FileOutputStream(file)), true, "UTF8");
- else
- return new PrintStream(new FileOutputStream(file), true, "UTF8");
- }
-
- public static Scanner openInFile(String filename)
- {
- Scanner localsc=null;
- try
- {
- localsc=new Scanner(new FileInputStream(filename), "UTF8");
-
- }catch(IOException ioe){
- System.out.println(ioe.getMessage());
- }
- return localsc;
- }
-
- public static FileInputStream openInputStream(String infilename)
- {
- FileInputStream fis=null;
- try {
- fis = new FileInputStream(infilename);
-
- } catch (IOException ioe) {
- System.out.println(ioe.getMessage());
- }
- return fis;
- }
-}
diff --git a/gi/posterior-regularisation/prjava/src/io/SerializedObjects.java b/gi/posterior-regularisation/prjava/src/io/SerializedObjects.java deleted file mode 100644 index d1631b51..00000000 --- a/gi/posterior-regularisation/prjava/src/io/SerializedObjects.java +++ /dev/null @@ -1,83 +0,0 @@ -package io; - - - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInput; -import java.io.ObjectInputStream; -import java.io.ObjectOutput; -import java.io.ObjectOutputStream; -import java.io.OutputStream; - -public class SerializedObjects -{ - public static void writeSerializedObject(Object object, String outFile) - { - ObjectOutput output = null; - try{ - //use buffering - OutputStream file = new FileOutputStream(outFile); - OutputStream buffer = new BufferedOutputStream( file ); - output = new ObjectOutputStream( buffer ); - output.writeObject(object); - buffer.close(); - file.close(); - } - catch(IOException ex){ - ex.printStackTrace(); - } - finally{ - try { - if (output != null) { - //flush and close "output" and its underlying streams - output.close(); - } - } - catch (IOException ex ){ - ex.printStackTrace(); - } - } - } - - public static Object readSerializedObject(String inputFile) - { - ObjectInput input = null; - Object recoveredObject=null; - try{ - //use buffering - InputStream file = new FileInputStream(inputFile); - InputStream buffer = new BufferedInputStream(file); - input = new ObjectInputStream(buffer); - //deserialize the List - recoveredObject = input.readObject(); - } - catch(IOException ex){ - ex.printStackTrace(); - } - catch (ClassNotFoundException ex){ - ex.printStackTrace(); - } - catch(Exception ex) - { - ex.printStackTrace(); - } - finally{ - try { - if ( input != null ) { - //close "input" and its underlying streams - input.close(); - } - } - catch (IOException ex){ - ex.printStackTrace(); - } - } - return recoveredObject; - } - -}
\ No newline at end of file diff --git a/gi/posterior-regularisation/prjava/src/optimization/examples/GeneralizedRosenbrock.java b/gi/posterior-regularisation/prjava/src/optimization/examples/GeneralizedRosenbrock.java deleted file mode 100644 index 25fa7f09..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/examples/GeneralizedRosenbrock.java +++ /dev/null @@ -1,110 +0,0 @@ -package optimization.examples; - - -import optimization.gradientBasedMethods.ConjugateGradient; -import optimization.gradientBasedMethods.GradientDescent; -import optimization.gradientBasedMethods.LBFGS; -import optimization.gradientBasedMethods.Objective; -import optimization.gradientBasedMethods.Optimizer; -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.linesearch.ArmijoLineSearchMinimization; -import optimization.linesearch.LineSearchMethod; -import optimization.stopCriteria.GradientL2Norm; -import optimization.stopCriteria.StopingCriteria; -import optimization.util.MathUtils; - -/** - * - * @author javg - * f(x) = \sum_{i=1}^{N-1} \left[ (1-x_i)^2+ 100 (x_{i+1} - x_i^2 )^2 \right] \quad \forall x\in\mathbb{R}^N. - */ -public class GeneralizedRosenbrock extends Objective{ - - - - public GeneralizedRosenbrock(int dimensions){ - parameters = new double[dimensions]; - java.util.Arrays.fill(parameters, 0); - gradient = new double[dimensions]; - - } - - public GeneralizedRosenbrock(int dimensions, double[] params){ - parameters = params; - gradient = new double[dimensions]; - } - - - public double getValue() { - functionCalls++; - double value = 0; - for(int i = 0; i < parameters.length-1; i++){ - value += MathUtils.square(1-parameters[i]) + 100*MathUtils.square(parameters[i+1] - MathUtils.square(parameters[i])); - } - - return value; - } - - /** - * gx = -2(1-x) -2x200(y-x^2) - * gy = 200(y-x^2) - */ - public double[] getGradient() { - gradientCalls++; - java.util.Arrays.fill(gradient,0); - for(int i = 0; i < parameters.length-1; i++){ - gradient[i]+=-2*(1-parameters[i]) - 400*parameters[i]*(parameters[i+1] - MathUtils.square(parameters[i])); - gradient[i+1]+=200*(parameters[i+1] - MathUtils.square(parameters[i])); - } - return gradient; - } - - - - - - - - public String toString(){ - String res =""; - for(int i = 0; i < parameters.length; i++){ - res += "P" + i+ " " + parameters[i]; - } - res += " Value " + getValue(); - return res; - } - - public static void main(String[] args) { - - GeneralizedRosenbrock o = new GeneralizedRosenbrock(2); - System.out.println("Starting optimization " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]); - ; - - System.out.println("Doing Gradient descent"); - //LineSearchMethod wolfe = new WolfRuleLineSearch(new InterpolationPickFirstStep(1),100,0.001,0.1); - StopingCriteria stop = new GradientL2Norm(0.001); - LineSearchMethod ls = new ArmijoLineSearchMinimization(); - Optimizer optimizer = new GradientDescent(ls); - OptimizerStats stats = new OptimizerStats(); - optimizer.setMaxIterations(1000); - boolean succed = optimizer.optimize(o,stats, stop); - System.out.println("Suceess " + succed + "/n"+stats.prettyPrint(1)); - System.out.println("Doing Conjugate Gradient descent"); - o = new GeneralizedRosenbrock(2); - // wolfe = new WolfRuleLineSearch(new InterpolationPickFirstStep(1),100,0.001,0.1); - optimizer = new ConjugateGradient(ls); - stats = new OptimizerStats(); - optimizer.setMaxIterations(1000); - succed = optimizer.optimize(o,stats,stop); - System.out.println("Suceess " + succed + "/n"+stats.prettyPrint(1)); - System.out.println("Doing Quasi newton descent"); - o = new GeneralizedRosenbrock(2); - optimizer = new LBFGS(ls,10); - stats = new OptimizerStats(); - optimizer.setMaxIterations(1000); - succed = optimizer.optimize(o,stats,stop); - System.out.println("Suceess " + succed + "/n"+stats.prettyPrint(1)); - - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/examples/x2y2.java b/gi/posterior-regularisation/prjava/src/optimization/examples/x2y2.java deleted file mode 100644 index f087681e..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/examples/x2y2.java +++ /dev/null @@ -1,128 +0,0 @@ -package optimization.examples; - - -import optimization.gradientBasedMethods.ConjugateGradient; - -import optimization.gradientBasedMethods.GradientDescent; -import optimization.gradientBasedMethods.LBFGS; -import optimization.gradientBasedMethods.Objective; -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.linesearch.GenericPickFirstStep; -import optimization.linesearch.LineSearchMethod; -import optimization.linesearch.WolfRuleLineSearch; -import optimization.stopCriteria.GradientL2Norm; -import optimization.stopCriteria.StopingCriteria; - - -/** - * @author javg - * - */ -public class x2y2 extends Objective{ - - - //Implements function ax2+ by2 - double a, b; - public x2y2(double a, double b){ - this.a = a; - this.b = b; - parameters = new double[2]; - parameters[0] = 4; - parameters[1] = 4; - gradient = new double[2]; - } - - public double getValue() { - functionCalls++; - return a*parameters[0]*parameters[0]+b*parameters[1]*parameters[1]; - } - - public double[] getGradient() { - gradientCalls++; - gradient[0]=2*a*parameters[0]; - gradient[1]=2*b*parameters[1]; - return gradient; -// if(debugLevel >=2){ -// double[] numericalGradient = DebugHelpers.getNumericalGradient(this, parameters, 0.000001); -// for(int i = 0; i < parameters.length; i++){ -// double diff = Math.abs(gradient[i]-numericalGradient[i]); -// if(diff > 0.00001){ -// System.out.println("Numerical Gradient does not match"); -// System.exit(1); -// } -// } -// } - } - - - - public void optimizeWithGradientDescent(LineSearchMethod ls, OptimizerStats stats, x2y2 o){ - GradientDescent optimizer = new GradientDescent(ls); - StopingCriteria stop = new GradientL2Norm(0.001); -// optimizer.setGradientConvergenceValue(0.001); - optimizer.setMaxIterations(100); - boolean succed = optimizer.optimize(o,stats,stop); - System.out.println("Ended optimzation Gradient Descent\n" + stats.prettyPrint(1)); - System.out.println("Solution: " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]); - if(succed){ - System.out.println("Ended optimization in " + optimizer.getCurrentIteration()); - }else{ - System.out.println("Failed to optimize"); - } - } - - public void optimizeWithConjugateGradient(LineSearchMethod ls, OptimizerStats stats, x2y2 o){ - ConjugateGradient optimizer = new ConjugateGradient(ls); - StopingCriteria stop = new GradientL2Norm(0.001); - - optimizer.setMaxIterations(10); - boolean succed = optimizer.optimize(o,stats,stop); - System.out.println("Ended optimzation Conjugate Gradient\n" + stats.prettyPrint(1)); - System.out.println("Solution: " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]); - if(succed){ - System.out.println("Ended optimization in " + optimizer.getCurrentIteration()); - }else{ - System.out.println("Failed to optimize"); - } - } - - public void optimizeWithLBFGS(LineSearchMethod ls, OptimizerStats stats, x2y2 o){ - LBFGS optimizer = new LBFGS(ls,10); - StopingCriteria stop = new GradientL2Norm(0.001); - optimizer.setMaxIterations(10); - boolean succed = optimizer.optimize(o,stats,stop); - System.out.println("Ended optimzation LBFGS\n" + stats.prettyPrint(1)); - System.out.println("Solution: " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]); - if(succed){ - System.out.println("Ended optimization in " + optimizer.getCurrentIteration()); - }else{ - System.out.println("Failed to optimize"); - } - } - - public static void main(String[] args) { - x2y2 o = new x2y2(1,10); - System.out.println("Starting optimization " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]); - o.setDebugLevel(4); - LineSearchMethod wolfe = new WolfRuleLineSearch(new GenericPickFirstStep(1),0.001,0.9);; -// LineSearchMethod ls = new ArmijoLineSearchMinimization(); - OptimizerStats stats = new OptimizerStats(); - o.optimizeWithGradientDescent(wolfe, stats, o); - o = new x2y2(1,10); - System.out.println("Starting optimization " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]); -// ls = new ArmijoLineSearchMinimization(); - stats = new OptimizerStats(); - o.optimizeWithConjugateGradient(wolfe, stats, o); - o = new x2y2(1,10); - System.out.println("Starting optimization " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]); -// ls = new ArmijoLineSearchMinimization(); - stats = new OptimizerStats(); - o.optimizeWithLBFGS(wolfe, stats, o); - } - - public String toString(){ - return "P1: " + parameters[0] + " P2: " + parameters[1] + " value " + getValue(); - } - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/examples/x2y2WithConstraints.java b/gi/posterior-regularisation/prjava/src/optimization/examples/x2y2WithConstraints.java deleted file mode 100644 index 391775b7..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/examples/x2y2WithConstraints.java +++ /dev/null @@ -1,127 +0,0 @@ -package optimization.examples; - - -import optimization.gradientBasedMethods.ProjectedGradientDescent; -import optimization.gradientBasedMethods.ProjectedObjective; -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.linesearch.ArmijoLineSearchMinimizationAlongProjectionArc; -import optimization.linesearch.InterpolationPickFirstStep; -import optimization.linesearch.LineSearchMethod; -import optimization.projections.BoundsProjection; -import optimization.projections.Projection; -import optimization.projections.SimplexProjection; -import optimization.stopCriteria.CompositeStopingCriteria; -import optimization.stopCriteria.GradientL2Norm; -import optimization.stopCriteria.ProjectedGradientL2Norm; -import optimization.stopCriteria.StopingCriteria; -import optimization.stopCriteria.ValueDifference; - - -/** - * @author javg - * - * - *ax2+ b(y2 -displacement) - */ -public class x2y2WithConstraints extends ProjectedObjective{ - - - double a, b; - double dx; - double dy; - Projection projection; - - - public x2y2WithConstraints(double a, double b, double[] params, double dx, double dy, Projection proj){ - //projection = new BoundsProjection(0.2,Double.MAX_VALUE); - super(); - projection = proj; - this.a = a; - this.b = b; - this.dx = dx; - this.dy = dy; - setInitialParameters(params); - System.out.println("Function " +a+"(x-"+dx+")^2 + "+b+"(y-"+dy+")^2"); - System.out.println("Gradient " +(2*a)+"(x-"+dx+") ; "+(b*2)+"(y-"+dy+")"); - printParameters(); - projection.project(parameters); - printParameters(); - gradient = new double[2]; - } - - public double getValue() { - functionCalls++; - return a*(parameters[0]-dx)*(parameters[0]-dx)+b*((parameters[1]-dy)*(parameters[1]-dy)); - } - - public double[] getGradient() { - if(gradient == null){ - gradient = new double[2]; - } - gradientCalls++; - gradient[0]=2*a*(parameters[0]-dx); - gradient[1]=2*b*(parameters[1]-dy); - return gradient; - } - - - public double[] projectPoint(double[] point) { - double[] newPoint = point.clone(); - projection.project(newPoint); - return newPoint; - } - - public void optimizeWithProjectedGradientDescent(LineSearchMethod ls, OptimizerStats stats, x2y2WithConstraints o){ - ProjectedGradientDescent optimizer = new ProjectedGradientDescent(ls); - StopingCriteria stopGrad = new ProjectedGradientL2Norm(0.001); - StopingCriteria stopValue = new ValueDifference(0.001); - CompositeStopingCriteria compositeStop = new CompositeStopingCriteria(); - compositeStop.add(stopGrad); - compositeStop.add(stopValue); - - optimizer.setMaxIterations(5); - boolean succed = optimizer.optimize(o,stats,compositeStop); - System.out.println("Ended optimzation Projected Gradient Descent\n" + stats.prettyPrint(1)); - System.out.println("Solution: " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]); - if(succed){ - System.out.println("Ended optimization in " + optimizer.getCurrentIteration()); - }else{ - System.out.println("Failed to optimize"); - } - } - - - - public String toString(){ - - return "P1: " + parameters[0] + " P2: " + parameters[1] + " value " + getValue() + " grad (" + getGradient()[0] + ":" + getGradient()[1]+")"; - } - - public static void main(String[] args) { - double a = 1; - double b=1; - double x0 = 0; - double y0 =1; - double dx = 0.5; - double dy = 0.5 ; - double [] parameters = new double[2]; - parameters[0] = x0; - parameters[1] = y0; - x2y2WithConstraints o = new x2y2WithConstraints(a,b,parameters,dx,dy, new SimplexProjection(0.5)); - System.out.println("Starting optimization " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1] + " a " + a + " b "+b ); - o.setDebugLevel(4); - - LineSearchMethod ls = new ArmijoLineSearchMinimizationAlongProjectionArc(new InterpolationPickFirstStep(1)); - - OptimizerStats stats = new OptimizerStats(); - o.optimizeWithProjectedGradientDescent(ls, stats, o); - -// o = new x2y2WithConstraints(a,b,x0,y0,dx,dy); -// stats = new OptimizerStats(); -// o.optimizeWithSpectralProjectedGradientDescent(stats, o); - } - - - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/AbstractGradientBaseMethod.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/AbstractGradientBaseMethod.java deleted file mode 100644 index 2fcb7990..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/AbstractGradientBaseMethod.java +++ /dev/null @@ -1,120 +0,0 @@ -package optimization.gradientBasedMethods; - -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.linesearch.DifferentiableLineSearchObjective; -import optimization.linesearch.LineSearchMethod; -import optimization.stopCriteria.StopingCriteria; -import optimization.util.MathUtils; - -/** - * - * @author javg - * - */ -public abstract class AbstractGradientBaseMethod implements Optimizer{ - - protected int maxNumberOfIterations=10000; - - - - protected int currentProjectionIteration; - protected double currValue; - protected double previousValue = Double.MAX_VALUE;; - protected double step; - protected double[] gradient; - public double[] direction; - - //Original values - protected double originalGradientL2Norm; - - protected LineSearchMethod lineSearch; - DifferentiableLineSearchObjective lso; - - - public void reset(){ - direction = null; - gradient = null; - previousValue = Double.MAX_VALUE; - currentProjectionIteration = 0; - originalGradientL2Norm = 0; - step = 0; - currValue = 0; - } - - public void initializeStructures(Objective o,OptimizerStats stats, StopingCriteria stop){ - lso = new DifferentiableLineSearchObjective(o); - } - public void updateStructuresBeforeStep(Objective o,OptimizerStats stats, StopingCriteria stop){ - } - - public void updateStructuresAfterStep(Objective o,OptimizerStats stats, StopingCriteria stop){ - } - - public boolean optimize(Objective o,OptimizerStats stats, StopingCriteria stop){ - //Initialize structures - - stats.collectInitStats(this, o); - direction = new double[o.getNumParameters()]; - initializeStructures(o, stats, stop); - for (currentProjectionIteration = 1; currentProjectionIteration < maxNumberOfIterations; currentProjectionIteration++){ - //System.out.println("\tgradient descent iteration " + currentProjectionIteration); - //System.out.print("\tparameters:" ); - //o.printParameters(); - previousValue = currValue; - currValue = o.getValue(); - gradient = o.getGradient(); - if(stop.stopOptimization(o)){ - stats.collectFinalStats(this, o); - return true; - } - - getDirection(); - if(MathUtils.dotProduct(gradient, direction) > 0){ - System.out.println("Not a descent direction"); - System.out.println(" current stats " + stats.prettyPrint(1)); - System.exit(-1); - } - updateStructuresBeforeStep(o, stats, stop); - lso.reset(direction); - step = lineSearch.getStepSize(lso); - //System.out.println("\t\tLeave with step: " + step); - if(step==-1){ - System.out.println("Failed to find step"); - stats.collectFinalStats(this, o); - return false; - } - updateStructuresAfterStep( o, stats, stop); -// previousValue = currValue; -// currValue = o.getValue(); -// gradient = o.getGradient(); - stats.collectIterationStats(this, o); - } - stats.collectFinalStats(this, o); - return false; - } - - - public int getCurrentIteration() { - return currentProjectionIteration; - } - - - /** - * Method specific - */ - public abstract double[] getDirection(); - - public double getCurrentStep() { - return step; - } - - - - public void setMaxIterations(int max) { - maxNumberOfIterations = max; - } - - public double getCurrentValue() { - return currValue; - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ConjugateGradient.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ConjugateGradient.java deleted file mode 100644 index 28295729..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ConjugateGradient.java +++ /dev/null @@ -1,92 +0,0 @@ -package optimization.gradientBasedMethods; - -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.linesearch.DifferentiableLineSearchObjective; -import optimization.linesearch.LineSearchMethod; -import optimization.stopCriteria.StopingCriteria; -import optimization.util.MathUtils; - - - -public class ConjugateGradient extends AbstractGradientBaseMethod{ - - - double[] previousGradient; - double[] previousDirection; - - public ConjugateGradient(LineSearchMethod lineSearch) { - this.lineSearch = lineSearch; - } - - public void reset(){ - super.reset(); - java.util.Arrays.fill(previousDirection, 0); - java.util.Arrays.fill(previousGradient, 0); - } - - public void initializeStructures(Objective o,OptimizerStats stats, StopingCriteria stop){ - super.initializeStructures(o, stats, stop); - previousGradient = new double[o.getNumParameters()]; - previousDirection = new double[o.getNumParameters()]; - } - public void updateStructuresBeforeStep(Objective o,OptimizerStats stats, StopingCriteria stop){ - System.arraycopy(gradient, 0, previousGradient, 0, gradient.length); - System.arraycopy(direction, 0, previousDirection, 0, direction.length); - } - -// public boolean optimize(Objective o,OptimizerStats stats, StopingCriteria stop){ -// DifferentiableLineSearchObjective lso = new DifferentiableLineSearchObjective(o); -// stats.collectInitStats(this, o); -// direction = new double[o.getNumParameters()]; -// initializeStructures(o, stats, stop); -// for (currentProjectionIteration = 0; currentProjectionIteration < maxNumberOfIterations; currentProjectionIteration++){ -// previousValue = currValue; -// currValue = o.getValue(); -// gradient =o.getGradient(); -// if(stop.stopOptimization(gradient)){ -// stats.collectFinalStats(this, o); -// return true; -// } -// getDirection(); -// updateStructures(o, stats, stop); -// lso.reset(direction); -// step = lineSearch.getStepSize(lso); -// if(step==-1){ -// System.out.println("Failed to find a step size"); -// System.out.println("Failed to find step"); -// stats.collectFinalStats(this, o); -// return false; -// } -// -// stats.collectIterationStats(this, o); -// } -// stats.collectFinalStats(this, o); -// return false; -// } - - public double[] getDirection(){ - direction = MathUtils.negation(gradient); - if(currentProjectionIteration != 1){ - //Using Polak-Ribiere method (book equation 5.45) - double b = MathUtils.dotProduct(gradient, MathUtils.arrayMinus(gradient, previousGradient)) - /MathUtils.dotProduct(previousGradient, previousGradient); - if(b<0){ - System.out.println("Defaulting to gradient descent"); - b = Math.max(b, 0); - } - MathUtils.plusEquals(direction, previousDirection, b); - //Debug code - if(MathUtils.dotProduct(direction, gradient) > 0){ - System.out.println("Not an descent direction reseting to gradien"); - direction = MathUtils.negation(gradient); - } - } - return direction; - } - - - - - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/DebugHelpers.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/DebugHelpers.java deleted file mode 100644 index 6dc4ef6c..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/DebugHelpers.java +++ /dev/null @@ -1,65 +0,0 @@ -package optimization.gradientBasedMethods; - -import java.util.ArrayList; - -import optimization.util.MathUtils; - - - -public class DebugHelpers { - public static void getLineSearchGraph(Objective o, double[] direction, - double[] parameters, double originalObj, - double originalDot, double c1, double c2){ - ArrayList<Double> stepS = new ArrayList<Double>(); - ArrayList<Double> obj = new ArrayList<Double>(); - ArrayList<Double> norm = new ArrayList<Double>(); - double[] gradient = new double[o.getNumParameters()]; - double[] newParameters = parameters.clone(); - MathUtils.plusEquals(newParameters,direction,0); - o.setParameters(newParameters); - double minValue = o.getValue(); - int valuesBiggerThanMax = 0; - for(double step = 0; step < 2; step +=0.01 ){ - newParameters = parameters.clone(); - MathUtils.plusEquals(newParameters,direction,step); - o.setParameters(newParameters); - double newValue = o.getValue(); - gradient = o.getGradient(); - double newgradDirectionDot = MathUtils.dotProduct(gradient,direction); - stepS.add(step); - obj.add(newValue); - norm.add(newgradDirectionDot); - if(newValue <= minValue){ - minValue = newValue; - }else{ - valuesBiggerThanMax++; - } - - if(valuesBiggerThanMax > 10){ - break; - } - - } - System.out.println("step\torigObj\tobj\tsuffdec\tnorm\tcurvature1"); - for(int i = 0; i < stepS.size(); i++){ - double cnorm= norm.get(i); - System.out.println(stepS.get(i)+"\t"+originalObj +"\t"+obj.get(i) + "\t" + - (originalObj + originalDot*((Double)stepS.get(i))*c1) +"\t"+Math.abs(cnorm) +"\t"+c2*Math.abs(originalDot)); - } - } - - public static double[] getNumericalGradient(Objective o, double[] parameters, double epsilon){ - int nrParameters = o.getNumParameters(); - double[] gradient = new double[nrParameters]; - double[] newParameters; - double originalValue = o.getValue(); - for(int parameter = 0; parameter < nrParameters; parameter++){ - newParameters = parameters.clone(); - newParameters[parameter]+=epsilon; - o.setParameters(newParameters); - double newValue = o.getValue(); - gradient[parameter]=(newValue-originalValue)/epsilon; - } - return gradient; - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/GradientDescent.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/GradientDescent.java deleted file mode 100644 index 9a53cef4..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/GradientDescent.java +++ /dev/null @@ -1,19 +0,0 @@ -package optimization.gradientBasedMethods; - -import optimization.linesearch.LineSearchMethod; - - - -public class GradientDescent extends AbstractGradientBaseMethod{ - - public GradientDescent(LineSearchMethod lineSearch) { - this.lineSearch = lineSearch; - } - - public double[] getDirection(){ - for(int i = 0; i< gradient.length; i++){ - direction[i] = -gradient[i]; - } - return direction; - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/LBFGS.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/LBFGS.java deleted file mode 100644 index dedbc942..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/LBFGS.java +++ /dev/null @@ -1,234 +0,0 @@ -package optimization.gradientBasedMethods; - - -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.linesearch.DifferentiableLineSearchObjective; -import optimization.linesearch.LineSearchMethod; -import optimization.stopCriteria.StopingCriteria; -import optimization.util.MathUtils; - -public class LBFGS extends AbstractGradientBaseMethod{ - - //How many previous values are being saved - int history; - double[][] skList; - double[][] ykList; - double initialHessianParameters; - double[] previousGradient; - double[] previousParameters; - - //auxiliar structures - double q[]; - double[] roi; - double[] alphai; - - public LBFGS(LineSearchMethod ls, int history) { - lineSearch = ls; - this.history = history; - skList = new double[history][]; - ykList = new double[history][]; - - } - - public void reset(){ - super.reset(); - initialHessianParameters = 0; - previousParameters = null; - previousGradient = null; - skList = new double[history][]; - ykList = new double[history][]; - q = null; - roi = null; - alphai = null; - } - - public double[] LBFGSTwoLoopRecursion(double hessianConst){ - //Only create array once - if(q == null){ - q = new double[gradient.length]; - } - System.arraycopy(gradient, 0, q, 0, gradient.length); - //Only create array once - if(roi == null){ - roi = new double[history]; - } - //Only create array once - if(alphai == null){ - alphai = new double[history]; - } - - for(int i = history-1; i >=0 && skList[i]!= null && ykList[i]!=null; i-- ){ - // System.out.println("New to Old proj " + currentProjectionIteration + " history "+history + " index " + i); - double[] si = skList[i]; - double[] yi = ykList[i]; - roi[i]= 1.0/MathUtils.dotProduct(yi,si); - alphai[i] = MathUtils.dotProduct(si, q)*roi[i]; - MathUtils.plusEquals(q, yi, -alphai[i]); - } - //Initial Hessian is just a constant - MathUtils.scalarMultiplication(q, hessianConst); - for(int i = 0; i <history && skList[i]!= null && ykList[i]!=null; i++ ){ - // System.out.println("Old to New proj " + currentProjectionIteration + " history "+history + " index " + i); - double beta = MathUtils.dotProduct(ykList[i], q)*roi[i]; - MathUtils.plusEquals(q, skList[i], (alphai[i]-beta)); - } - return q; - } - - - - - @Override - public double[] getDirection() { - - calculateInitialHessianParameter(); -// System.out.println("Initial hessian " + initialHessianParameters); - return direction = MathUtils.negation(LBFGSTwoLoopRecursion(initialHessianParameters)); - } - - public void calculateInitialHessianParameter(){ - if(currentProjectionIteration == 1){ - //Use gradient - initialHessianParameters = 1; - }else if(currentProjectionIteration <= history){ - double[] sk = skList[currentProjectionIteration-2]; - double[] yk = ykList[currentProjectionIteration-2]; - initialHessianParameters = MathUtils.dotProduct(sk, yk)/MathUtils.dotProduct(yk, yk); - }else{ - //get the last one - double[] sk = skList[history-1]; - double[] yk = ykList[history-1]; - initialHessianParameters = MathUtils.dotProduct(sk, yk)/MathUtils.dotProduct(yk, yk); - } - } - - //TODO if structures exit just reset them to zero - public void initializeStructures(Objective o,OptimizerStats stats, StopingCriteria stop){ - super.initializeStructures(o, stats, stop); - previousParameters = new double[o.getNumParameters()]; - previousGradient = new double[o.getNumParameters()]; - } - public void updateStructuresBeforeStep(Objective o,OptimizerStats stats, StopingCriteria stop){ - super.initializeStructures(o, stats, stop); - System.arraycopy(o.getParameters(), 0, previousParameters, 0, previousParameters.length); - System.arraycopy(gradient, 0, previousGradient, 0, gradient.length); - } - - public void updateStructuresAfterStep( Objective o,OptimizerStats stats, StopingCriteria stop){ - double[] diffX = MathUtils.arrayMinus(o.getParameters(), previousParameters); - double[] diffGrad = MathUtils.arrayMinus(gradient, previousGradient); - //Save new values and discard new ones - if(currentProjectionIteration > history){ - for(int i = 0; i < history-1;i++){ - skList[i]=skList[i+1]; - ykList[i]=ykList[i+1]; - } - skList[history-1]=diffX; - ykList[history-1]=diffGrad; - }else{ - skList[currentProjectionIteration-1]=diffX; - ykList[currentProjectionIteration-1]=diffGrad; - } - } - -// public boolean optimize(Objective o, OptimizerStats stats, StopingCriteria stop) { -// DifferentiableLineSearchObjective lso = new DifferentiableLineSearchObjective(o); -// gradient = o.getGradient(); -// direction = new double[o.getNumParameters()]; -// previousGradient = new double[o.getNumParameters()]; -// -// previousParameters = new double[o.getNumParameters()]; -// -// stats.collectInitStats(this, o); -// previousValue = Double.MAX_VALUE; -// currValue= o.getValue(); -// //Used for stopping criteria -// double[] originalGradient = o.getGradient(); -// -// originalGradientL2Norm = MathUtils.L2Norm(originalGradient); -// if(stop.stopOptimization(originalGradient)){ -// stats.collectFinalStats(this, o); -// return true; -// } -// for (currentProjectionIteration = 1; currentProjectionIteration < maxNumberOfIterations; currentProjectionIteration++){ -// -// -// currValue = o.getValue(); -// gradient = o.getGradient(); -// currParameters = o.getParameters(); -// -// -// if(currentProjectionIteration == 1){ -// //Use gradient -// initialHessianParameters = 1; -// }else if(currentProjectionIteration <= history){ -// double[] sk = skList[currentProjectionIteration-2]; -// double[] yk = ykList[currentProjectionIteration-2]; -// initialHessianParameters = MathUtils.dotProduct(sk, yk)/MathUtils.dotProduct(yk, yk); -// }else{ -// //get the last one -// double[] sk = skList[history-1]; -// double[] yk = ykList[history-1]; -// initialHessianParameters = MathUtils.dotProduct(sk, yk)/MathUtils.dotProduct(yk, yk); -// } -// -// getDirection(); -// -// //MatrixOutput.printDoubleArray(direction, "direction"); -// double dot = MathUtils.dotProduct(direction, gradient); -// if(dot > 0){ -// throw new RuntimeException("Not a descent direction"); -// } if (Double.isNaN(dot)){ -// throw new RuntimeException("dot is not a number!!"); -// } -// System.arraycopy(currParameters, 0, previousParameters, 0, currParameters.length); -// System.arraycopy(gradient, 0, previousGradient, 0, gradient.length); -// lso.reset(direction); -// step = lineSearch.getStepSize(lso); -// if(step==-1){ -// System.out.println("Failed to find a step size"); -//// lso.printLineSearchSteps(); -//// System.out.println(stats.prettyPrint(1)); -// stats.collectFinalStats(this, o); -// return false; -// } -// stats.collectIterationStats(this, o); -// -// //We are not updating the alpha since it is done in line search already -// currParameters = o.getParameters(); -// gradient = o.getGradient(); -// -// if(stop.stopOptimization(gradient)){ -// stats.collectFinalStats(this, o); -// return true; -// } -// double[] diffX = MathUtils.arrayMinus(currParameters, previousParameters); -// double[] diffGrad = MathUtils.arrayMinus(gradient, previousGradient); -// //Save new values and discard new ones -// if(currentProjectionIteration > history){ -// for(int i = 0; i < history-1;i++){ -// skList[i]=skList[i+1]; -// ykList[i]=ykList[i+1]; -// } -// skList[history-1]=diffX; -// ykList[history-1]=diffGrad; -// }else{ -// skList[currentProjectionIteration-1]=diffX; -// ykList[currentProjectionIteration-1]=diffGrad; -// } -// previousValue = currValue; -// } -// stats.collectFinalStats(this, o); -// return false; -// } - - - - - - - - - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/Objective.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/Objective.java deleted file mode 100644 index 6be01bf9..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/Objective.java +++ /dev/null @@ -1,87 +0,0 @@ -package optimization.gradientBasedMethods; - - -/** - * Defines an optimization objective: - * - * - * @author javg - * - */ -public abstract class Objective { - - protected int functionCalls = 0; - protected int gradientCalls = 0; - protected int updateCalls = 0; - - protected double[] parameters; - - //Contains a cache with the gradient - public double[] gradient; - int debugLevel = 0; - - public void setDebugLevel(int level){ - debugLevel = level; - } - - public int getNumParameters() { - return parameters.length; - } - - public double getParameter(int index) { - return parameters[index]; - } - - public double[] getParameters() { - return parameters; - } - - public abstract double[] getGradient( ); - - public void setParameter(int index, double value) { - parameters[index]=value; - } - - public void setParameters(double[] params) { - if(parameters == null){ - parameters = new double[params.length]; - } - updateCalls++; - System.arraycopy(params, 0, parameters, 0, params.length); - } - - - public int getNumberFunctionCalls() { - return functionCalls; - } - - public int getNumberGradientCalls() { - return gradientCalls; - } - - public int getNumberUpdateCalls() { - return updateCalls; - } - - public String finalInfoString() { - return "FE: " + functionCalls + " GE " + gradientCalls + " Params updates" + - updateCalls; - } - public void printParameters() { - System.out.println(toString()); - } - - public abstract String toString(); - public abstract double getValue (); - - /** - * Sets the initial objective parameters - * For unconstrained models this just sets the objective params = argument no copying - * For a constrained objective project the parameters and then set - * @param params - */ - public void setInitialParameters(double[] params){ - parameters = params; - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/Optimizer.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/Optimizer.java deleted file mode 100644 index 96fce5b0..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/Optimizer.java +++ /dev/null @@ -1,19 +0,0 @@ -package optimization.gradientBasedMethods; - -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.stopCriteria.StopingCriteria; - -public interface Optimizer { - public boolean optimize(Objective o,OptimizerStats stats, StopingCriteria stoping); - - - public double[] getDirection(); - public double getCurrentStep(); - public double getCurrentValue(); - public int getCurrentIteration(); - public void reset(); - - public void setMaxIterations(int max); - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedAbstractGradientBaseMethod.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedAbstractGradientBaseMethod.java deleted file mode 100644 index afb29d04..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedAbstractGradientBaseMethod.java +++ /dev/null @@ -1,11 +0,0 @@ -package optimization.gradientBasedMethods; - - -/** - * - * @author javg - * - */ -public abstract class ProjectedAbstractGradientBaseMethod extends AbstractGradientBaseMethod implements ProjectedOptimizer{ - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedGradientDescent.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedGradientDescent.java deleted file mode 100644 index 0186e945..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedGradientDescent.java +++ /dev/null @@ -1,154 +0,0 @@ -package optimization.gradientBasedMethods; - -import java.io.IOException; - -import optimization.gradientBasedMethods.stats.OptimizerStats; -import optimization.linesearch.DifferentiableLineSearchObjective; -import optimization.linesearch.LineSearchMethod; -import optimization.linesearch.ProjectedDifferentiableLineSearchObjective; -import optimization.stopCriteria.StopingCriteria; -import optimization.util.MathUtils; - - -/** - * This class implements the projected gradiend - * as described in Bertsekas "Non Linear Programming" - * section 2.3. - * - * The update is given by: - * x_k+1 = x_k + alpha^k(xbar_k-x_k) - * Where xbar is: - * xbar = [x_k -s_k grad(f(x_k))]+ - * where []+ is the projection into the feasibility set - * - * alpha is the step size - * s_k - is a positive scalar which can be view as a step size as well, by - * setting alpha to 1, then x_k+1 = [x_k -s_k grad(f(x_k))]+ - * This is called taking a step size along the projection arc (Bertsekas) which - * we will use by default. - * - * Note that the only place where we actually take a step size is on pick a step size - * so this is going to be just like a normal gradient descent but use a different - * armijo line search where we project after taking a step. - * - * - * @author javg - * - */ -public class ProjectedGradientDescent extends ProjectedAbstractGradientBaseMethod{ - - - - - public ProjectedGradientDescent(LineSearchMethod lineSearch) { - this.lineSearch = lineSearch; - } - - //Use projected differential objective instead - public void initializeStructures(Objective o, OptimizerStats stats, StopingCriteria stop) { - lso = new ProjectedDifferentiableLineSearchObjective(o); - }; - - - ProjectedObjective obj; - public boolean optimize(ProjectedObjective o,OptimizerStats stats, StopingCriteria stop){ - obj = o; - return super.optimize(o, stats, stop); - } - - public double[] getDirection(){ - for(int i = 0; i< gradient.length; i++){ - direction[i] = -gradient[i]; - } - return direction; - } - - - - -} - - - - - - - -///OLD CODE - -//Use projected gradient norm -//public boolean stopCriteria(double[] gradient){ -// if(originalDirenctionL2Norm == 0){ -// System.out.println("Leaving original direction norm is zero"); -// return true; -// } -// if(MathUtils.L2Norm(direction)/originalDirenctionL2Norm < gradientConvergenceValue){ -// System.out.println("Leaving projected gradient Norm smaller than epsilon"); -// return true; -// } -// if((previousValue - currValue)/Math.abs(previousValue) < valueConvergenceValue) { -// System.out.println("Leaving value change below treshold " + previousValue + " - " + currValue); -// System.out.println(previousValue/currValue + " - " + currValue/currValue -// + " = " + (previousValue - currValue)/Math.abs(previousValue)); -// return true; -// } -// return false; -//} -// - -//public boolean optimize(ProjectedObjective o,OptimizerStats stats, StopingCriteria stop){ -// stats.collectInitStats(this, o); -// obj = o; -// step = 0; -// currValue = o.getValue(); -// previousValue = Double.MAX_VALUE; -// gradient = o.getGradient(); -// originalGradientL2Norm = MathUtils.L2Norm(gradient); -// parameterChange = new double[gradient.length]; -// getDirection(); -// ProjectedDifferentiableLineSearchObjective lso = new ProjectedDifferentiableLineSearchObjective(o,direction); -// -// originalDirenctionL2Norm = MathUtils.L2Norm(direction); -// //MatrixOutput.printDoubleArray(currParameters, "parameters"); -// for (currentProjectionIteration = 0; currentProjectionIteration < maxNumberOfIterations; currentProjectionIteration++){ -// // System.out.println("Iter " + currentProjectionIteration); -// //o.printParameters(); -// -// -// -// if(stop.stopOptimization(gradient)){ -// stats.collectFinalStats(this, o); -// lastStepUsed = step; -// return true; -// } -// lso.reset(direction); -// step = lineSearch.getStepSize(lso); -// if(step==-1){ -// System.out.println("Failed to find step"); -// stats.collectFinalStats(this, o); -// return false; -// -// } -// -// //Update the direction for stopping criteria -// previousValue = currValue; -// currValue = o.getValue(); -// gradient = o.getGradient(); -// direction = getDirection(); -// if(MathUtils.dotProduct(gradient, direction) > 0){ -// System.out.println("Not a descent direction"); -// System.out.println(" current stats " + stats.prettyPrint(1)); -// System.exit(-1); -// } -// stats.collectIterationStats(this, o); -// } -// lastStepUsed = step; -// stats.collectFinalStats(this, o); -// return false; -// } - -//public boolean optimize(Objective o,OptimizerStats stats, StopingCriteria stop){ -// System.out.println("Objective is not a projected objective"); -// throw new RuntimeException(); -//} - diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedObjective.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedObjective.java deleted file mode 100644 index c3d21393..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedObjective.java +++ /dev/null @@ -1,29 +0,0 @@ -package optimization.gradientBasedMethods; - -import optimization.util.MathUtils; - - -/** - * Computes a projected objective - * When we tell it to set some parameters it automatically projects the parameters back into the simplex: - * - * - * When we tell it to get the gradient in automatically returns the projected gradient: - * @author javg - * - */ -public abstract class ProjectedObjective extends Objective{ - - public abstract double[] projectPoint (double[] point); - - public double[] auxParameters; - - - public void setInitialParameters(double[] params){ - setParameters(projectPoint(params)); - } - - - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedOptimizer.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedOptimizer.java deleted file mode 100644 index 81d8403e..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/ProjectedOptimizer.java +++ /dev/null @@ -1,10 +0,0 @@ -package optimization.gradientBasedMethods; - - - -public interface ProjectedOptimizer extends Optimizer{ - - - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/stats/OptimizerStats.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/stats/OptimizerStats.java deleted file mode 100644 index 6340ef73..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/stats/OptimizerStats.java +++ /dev/null @@ -1,86 +0,0 @@ -package optimization.gradientBasedMethods.stats; - -import java.util.ArrayList; - -import optimization.gradientBasedMethods.Objective; -import optimization.gradientBasedMethods.Optimizer; -import optimization.util.MathUtils; -import optimization.util.StaticTools; - - -public class OptimizerStats { - - double start = 0; - double totalTime = 0; - - String objectiveFinalStats; - - ArrayList<Double> gradientNorms = new ArrayList<Double>(); - ArrayList<Double> steps = new ArrayList<Double>(); - ArrayList<Double> value = new ArrayList<Double>(); - ArrayList<Integer> iterations = new ArrayList<Integer>(); - double prevValue =0; - - public void reset(){ - start = 0; - totalTime = 0; - - objectiveFinalStats=""; - - gradientNorms.clear(); - steps.clear(); - value.clear(); - iterations.clear(); - prevValue =0; - } - - public void startTime() { - start = System.currentTimeMillis(); - } - public void stopTime() { - totalTime += System.currentTimeMillis() - start; - } - - public String prettyPrint(int level){ - StringBuffer res = new StringBuffer(); - res.append("Total time " + totalTime/1000 + " seconds \n" + "Iterations " + iterations.size() + "\n"); - res.append(objectiveFinalStats+"\n"); - if(level > 0){ - if(iterations.size() > 0){ - res.append("\tIteration"+iterations.get(0)+"\tstep: "+StaticTools.prettyPrint(steps.get(0), "0.00E00", 6)+ "\tgradientNorm "+ - StaticTools.prettyPrint(gradientNorms.get(0), "0.00000E00", 10)+ "\tvalue "+ StaticTools.prettyPrint(value.get(0), "0.000000E00",11)+"\n"); - } - for(int i = 1; i < iterations.size(); i++){ - res.append("\tIteration:\t"+iterations.get(i)+"\tstep:"+StaticTools.prettyPrint(steps.get(i), "0.00E00", 6)+ "\tgradientNorm "+ - StaticTools.prettyPrint(gradientNorms.get(i), "0.00000E00", 10)+ - "\tvalue:\t"+ StaticTools.prettyPrint(value.get(i), "0.000000E00",11)+ - "\tvalueDiff:\t"+ StaticTools.prettyPrint((value.get(i-1)-value.get(i)), "0.000000E00",11)+ - "\n"); - } - } - return res.toString(); - } - - - public void collectInitStats(Optimizer optimizer, Objective objective){ - startTime(); - iterations.add(-1); - gradientNorms.add(MathUtils.L2Norm(objective.getGradient())); - steps.add(0.0); - value.add(objective.getValue()); - } - - public void collectIterationStats(Optimizer optimizer, Objective objective){ - iterations.add(optimizer.getCurrentIteration()); - gradientNorms.add(MathUtils.L2Norm(objective.getGradient())); - steps.add(optimizer.getCurrentStep()); - value.add(optimizer.getCurrentValue()); - } - - - public void collectFinalStats(Optimizer optimizer, Objective objective){ - stopTime(); - objectiveFinalStats = objective.finalInfoString(); - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/stats/ProjectedOptimizerStats.java b/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/stats/ProjectedOptimizerStats.java deleted file mode 100644 index d65a1267..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/gradientBasedMethods/stats/ProjectedOptimizerStats.java +++ /dev/null @@ -1,70 +0,0 @@ -package optimization.gradientBasedMethods.stats; - -import java.util.ArrayList; - -import optimization.gradientBasedMethods.Objective; -import optimization.gradientBasedMethods.Optimizer; -import optimization.gradientBasedMethods.ProjectedObjective; -import optimization.gradientBasedMethods.ProjectedOptimizer; -import optimization.util.MathUtils; -import optimization.util.StaticTools; - - -public class ProjectedOptimizerStats extends OptimizerStats{ - - - - public void reset(){ - super.reset(); - projectedGradientNorms.clear(); - } - - ArrayList<Double> projectedGradientNorms = new ArrayList<Double>(); - - public String prettyPrint(int level){ - StringBuffer res = new StringBuffer(); - res.append("Total time " + totalTime/1000 + " seconds \n" + "Iterations " + iterations.size() + "\n"); - res.append(objectiveFinalStats+"\n"); - if(level > 0){ - if(iterations.size() > 0){ - res.append("\tIteration"+iterations.get(0)+"\tstep: "+ - StaticTools.prettyPrint(steps.get(0), "0.00E00", 6)+ "\tgradientNorm "+ - StaticTools.prettyPrint(gradientNorms.get(0), "0.00000E00", 10) - + "\tdirection"+ - StaticTools.prettyPrint(projectedGradientNorms.get(0), "0.00000E00", 10)+ - "\tvalue "+ StaticTools.prettyPrint(value.get(0), "0.000000E00",11)+"\n"); - } - for(int i = 1; i < iterations.size(); i++){ - res.append("\tIteration"+iterations.get(i)+"\tstep: "+StaticTools.prettyPrint(steps.get(i), "0.00E00", 6)+ "\tgradientNorm "+ - StaticTools.prettyPrint(gradientNorms.get(i), "0.00000E00", 10)+ - "\t direction "+ - StaticTools.prettyPrint(projectedGradientNorms.get(i), "0.00000E00", 10)+ - "\tvalue "+ StaticTools.prettyPrint(value.get(i), "0.000000E00",11)+ - "\tvalueDiff "+ StaticTools.prettyPrint((value.get(i-1)-value.get(i)), "0.000000E00",11)+ - "\n"); - } - } - return res.toString(); - } - - - public void collectInitStats(Optimizer optimizer, Objective objective){ - startTime(); - } - - public void collectIterationStats(Optimizer optimizer, Objective objective){ - iterations.add(optimizer.getCurrentIteration()); - gradientNorms.add(MathUtils.L2Norm(objective.getGradient())); - projectedGradientNorms.add(MathUtils.L2Norm(optimizer.getDirection())); - steps.add(optimizer.getCurrentStep()); - value.add(optimizer.getCurrentValue()); - } - - - - public void collectFinalStats(Optimizer optimizer, Objective objective){ - stopTime(); - objectiveFinalStats = objective.finalInfoString(); - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/ArmijoLineSearchMinimization.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/ArmijoLineSearchMinimization.java deleted file mode 100644 index c9f9b8df..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/ArmijoLineSearchMinimization.java +++ /dev/null @@ -1,102 +0,0 @@ -package optimization.linesearch; - -import optimization.util.Interpolation; - - -/** - * Implements Back Tracking Line Search as described on page 37 of Numerical Optimization. - * Also known as armijo rule - * @author javg - * - */ -public class ArmijoLineSearchMinimization implements LineSearchMethod{ - - /** - * How much should the step size decrease at each iteration. - */ - double contractionFactor = 0.5; - double c1 = 0.0001; - - double sigma1 = 0.1; - double sigma2 = 0.9; - - - - double initialStep; - int maxIterations = 10; - - - public ArmijoLineSearchMinimization(){ - this.initialStep = 1; - } - - //Experiment - double previousStepPicked = -1;; - double previousInitGradientDot = -1; - double currentInitGradientDot = -1; - - - public void reset(){ - previousStepPicked = -1;; - previousInitGradientDot = -1; - currentInitGradientDot = -1; - } - - public void setInitialStep(double initial){ - initialStep = initial; - } - - /** - * - */ - - public double getStepSize(DifferentiableLineSearchObjective o) { - currentInitGradientDot = o.getInitialGradient(); - //Should update all in the objective - o.updateAlpha(initialStep); - int nrIterations = 0; - //System.out.println("tried alpha" + initialStep + " value " + o.getCurrentValue()); - while(!WolfeConditions.suficientDecrease(o,c1)){ - if(nrIterations >= maxIterations){ - o.printLineSearchSteps(); - return -1; - } - double alpha=o.getAlpha(); - double alphaTemp = - Interpolation.quadraticInterpolation(o.getOriginalValue(), o.getInitialGradient(), alpha, o.getCurrentValue()); - if(alphaTemp >= sigma1 || alphaTemp <= sigma2*o.getAlpha()){ -// System.out.println("using alpha temp " + alphaTemp); - alpha = alphaTemp; - }else{ -// System.out.println("Discarding alpha temp " + alphaTemp); - alpha = alpha*contractionFactor; - } -// double alpha =o.getAlpha()*contractionFactor; - - o.updateAlpha(alpha); - //System.out.println("tried alpha" + alpha+ " value " + o.getCurrentValue()); - nrIterations++; - } - - //System.out.println("Leavning line search used:"); - //o.printLineSearchSteps(); - - previousInitGradientDot = currentInitGradientDot; - previousStepPicked = o.getAlpha(); - return o.getAlpha(); - } - - public double getInitialGradient() { - return currentInitGradientDot; - - } - - public double getPreviousInitialGradient() { - return previousInitGradientDot; - } - - public double getPreviousStepUsed() { - return previousStepPicked; - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/ArmijoLineSearchMinimizationAlongProjectionArc.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/ArmijoLineSearchMinimizationAlongProjectionArc.java deleted file mode 100644 index e153f2da..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/ArmijoLineSearchMinimizationAlongProjectionArc.java +++ /dev/null @@ -1,141 +0,0 @@ -package optimization.linesearch; - -import optimization.gradientBasedMethods.ProjectedObjective; -import optimization.util.Interpolation; -import optimization.util.MathUtils; - - - - - -/** - * Implements Armijo Rule Line search along the projection arc (Non-Linear Programming page 230) - * To be used with Projected gradient Methods. - * - * Recall that armijo tries successive step sizes alpha until the sufficient decrease is satisfied: - * f(x+alpha*direction) < f(x) + alpha*c1*grad(f)*direction - * - * In this case we are optimizing over a convex set X so we must guarantee that the new point stays inside the - * constraints. - * First the direction as to be feasible (inside constraints) and will be define as: - * d = (x_k_f - x_k) where x_k_f is a feasible point. - * so the armijo condition can be rewritten as: - * f(x+alpha(x_k_f - x_k)) < f(x) + c1*grad(f)*(x_k_f - x_k) - * and x_k_f is defined as: - * [x_k-alpha*grad(f)]+ - * where []+ mean a projection to the feasibility set. - * So this means that we take a step on the negative gradient (gradient descent) and then obtain then project - * that point to the feasibility set. - * Note that if the point is already feasible then we are back to the normal armijo rule. - * - * @author javg - * - */ -public class ArmijoLineSearchMinimizationAlongProjectionArc implements LineSearchMethod{ - - /** - * How much should the step size decrease at each iteration. - */ - double contractionFactor = 0.5; - double c1 = 0.0001; - - - double initialStep; - int maxIterations = 100; - - - double sigma1 = 0.1; - double sigma2 = 0.9; - - //Experiment - double previousStepPicked = -1;; - double previousInitGradientDot = -1; - double currentInitGradientDot = -1; - - GenericPickFirstStep strategy; - - - public void reset(){ - previousStepPicked = -1;; - previousInitGradientDot = -1; - currentInitGradientDot = -1; - } - - - public ArmijoLineSearchMinimizationAlongProjectionArc(){ - this.initialStep = 1; - } - - public ArmijoLineSearchMinimizationAlongProjectionArc(GenericPickFirstStep strategy){ - this.strategy = strategy; - this.initialStep = strategy.getFirstStep(this); - } - - - public void setInitialStep(double initial){ - this.initialStep = initial; - } - - /** - * - */ - - public double getStepSize(DifferentiableLineSearchObjective o) { - - - //Should update all in the objective - initialStep = strategy.getFirstStep(this); - o.updateAlpha(initialStep); - previousInitGradientDot=currentInitGradientDot; - currentInitGradientDot=o.getCurrentGradient(); - int nrIterations = 0; - - //Armijo rule, the current value has to be smaller than the original value plus a small step of the gradient - while(o.getCurrentValue() > - o.getOriginalValue() + c1*(o.getCurrentGradient())){ -// System.out.println("curr value "+o.getCurrentValue()); -// System.out.println("original value "+o.getOriginalValue()); -// System.out.println("GRADIENT decrease" +(MathUtils.dotProduct(o.o.gradient, -// MathUtils.arrayMinus(o.originalParameters,((ProjectedObjective)o.o).auxParameters)))); -// System.out.println("GRADIENT SAVED" + o.getCurrentGradient()); - if(nrIterations >= maxIterations){ - System.out.println("Could not find a step leaving line search with -1"); - o.printLineSearchSteps(); - return -1; - } - double alpha=o.getAlpha(); - double alphaTemp = - Interpolation.quadraticInterpolation(o.getOriginalValue(), o.getInitialGradient(), alpha, o.getCurrentValue()); - if(alphaTemp >= sigma1 || alphaTemp <= sigma2*o.getAlpha()){ - alpha = alphaTemp; - }else{ - alpha = alpha*contractionFactor; - } -// double alpha =obj.getAlpha()*contractionFactor; - o.updateAlpha(alpha); - nrIterations++; - } -// System.out.println("curr value "+o.getCurrentValue()); -// System.out.println("original value "+o.getOriginalValue()); -// System.out.println("sufficient decrease" +c1*o.getCurrentGradient()); -// System.out.println("Leavning line search used:"); -// o.printSmallLineSearchSteps(); - - previousStepPicked = o.getAlpha(); - return o.getAlpha(); - } - - public double getInitialGradient() { - return currentInitGradientDot; - - } - - public double getPreviousInitialGradient() { - return previousInitGradientDot; - } - - public double getPreviousStepUsed() { - return previousStepPicked; - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/DifferentiableLineSearchObjective.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/DifferentiableLineSearchObjective.java deleted file mode 100644 index a5bc958e..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/DifferentiableLineSearchObjective.java +++ /dev/null @@ -1,185 +0,0 @@ -package optimization.linesearch; - -import gnu.trove.TDoubleArrayList; -import gnu.trove.TIntArrayList; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; - -import optimization.gradientBasedMethods.Objective; -import optimization.util.MathUtils; -import optimization.util.StaticTools; - - - -import util.MathUtil; -import util.Printing; - - -/** - * A wrapper class for the actual objective in order to perform - * line search. The optimization code assumes that this does a lot - * of caching in order to simplify legibility. For the applications - * we use it for, caching the entire history of evaluations should be - * a win. - * - * Note: the lastEvaluatedAt value is very important, since we will use - * it to avoid doing an evaluation of the gradient after the line search. - * - * The differentiable line search objective defines a search along the ray - * given by a direction of the main objective. - * It defines the following function, - * where f is the original objective function: - * g(alpha) = f(x_0 + alpha*direction) - * g'(alpha) = f'(x_0 + alpha*direction)*direction - * - * @author joao - * - */ -public class DifferentiableLineSearchObjective { - - - - Objective o; - int nrIterations; - TDoubleArrayList steps; - TDoubleArrayList values; - TDoubleArrayList gradients; - - //This variables cannot change - public double[] originalParameters; - public double[] searchDirection; - - - /** - * Defines a line search objective: - * Receives: - * Objective to each we are performing the line search, is used to calculate values and gradients - * Direction where to do the ray search, note that the direction does not depend of the - * objective but depends from the method. - * @param o - * @param direction - */ - public DifferentiableLineSearchObjective(Objective o) { - this.o = o; - originalParameters = new double[o.getNumParameters()]; - searchDirection = new double[o.getNumParameters()]; - steps = new TDoubleArrayList(); - values = new TDoubleArrayList(); - gradients = new TDoubleArrayList(); - } - /** - * Called whenever we start a new iteration. - * Receives the ray where we are searching for and resets all values - * - */ - public void reset(double[] direction){ - //Copy initial values - System.arraycopy(o.getParameters(), 0, originalParameters, 0, o.getNumParameters()); - System.arraycopy(direction, 0, searchDirection, 0, o.getNumParameters()); - - //Initialize variables - nrIterations = 0; - steps.clear(); - values.clear(); - gradients.clear(); - - values.add(o.getValue()); - gradients.add(MathUtils.dotProduct(o.getGradient(),direction)); - steps.add(0); - } - - - /** - * update the current value of alpha. - * Takes a step with that alpha in direction - * Get the real objective value and gradient and calculate all required information. - */ - public void updateAlpha(double alpha){ - if(alpha < 0){ - System.out.println("alpha may not be smaller that zero"); - throw new RuntimeException(); - } - nrIterations++; - steps.add(alpha); - //x_t+1 = x_t + alpha*direction - System.arraycopy(originalParameters,0, o.getParameters(), 0, originalParameters.length); - MathUtils.plusEquals(o.getParameters(), searchDirection, alpha); - o.setParameters(o.getParameters()); -// System.out.println("Took a step of " + alpha + " new value " + o.getValue()); - values.add(o.getValue()); - gradients.add(MathUtils.dotProduct(o.getGradient(),searchDirection)); - } - - - - public int getNrIterations(){ - return nrIterations; - } - - /** - * return g(alpha) for the current value of alpha - * @param iter - * @return - */ - public double getValue(int iter){ - return values.get(iter); - } - - public double getCurrentValue(){ - return values.get(nrIterations); - } - - public double getOriginalValue(){ - return values.get(0); - } - - /** - * return g'(alpha) for the current value of alpha - * @param iter - * @return - */ - public double getGradient(int iter){ - return gradients.get(iter); - } - - public double getCurrentGradient(){ - return gradients.get(nrIterations); - } - - public double getInitialGradient(){ - return gradients.get(0); - } - - - - - public double getAlpha(){ - return steps.get(nrIterations); - } - - public void printLineSearchSteps(){ - System.out.println( - " Steps size "+steps.size() + - "Values size "+values.size() + - "Gradeients size "+gradients.size()); - for(int i =0; i < steps.size();i++){ - System.out.println("Iter " + i + " step " + steps.get(i) + - " value " + values.get(i) + " grad " + gradients.get(i)); - } - } - - public void printSmallLineSearchSteps(){ - for(int i =0; i < steps.size();i++){ - System.out.print(StaticTools.prettyPrint(steps.get(i), "0.0000E00",8) + " "); - } - System.out.println(); - } - - public static void main(String[] args) { - - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/GenericPickFirstStep.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/GenericPickFirstStep.java deleted file mode 100644 index a33eb311..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/GenericPickFirstStep.java +++ /dev/null @@ -1,20 +0,0 @@ -package optimization.linesearch; - - -public class GenericPickFirstStep{ - double _initValue; - public GenericPickFirstStep(double initValue) { - _initValue = initValue; - } - - public double getFirstStep(LineSearchMethod ls){ - return _initValue; - } - public void collectInitValues(LineSearchMethod ls){ - - } - - public void collectFinalValues(LineSearchMethod ls){ - - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/InterpolationPickFirstStep.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/InterpolationPickFirstStep.java deleted file mode 100644 index 0deebcdb..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/InterpolationPickFirstStep.java +++ /dev/null @@ -1,25 +0,0 @@ -package optimization.linesearch; - - -public class InterpolationPickFirstStep extends GenericPickFirstStep{ - public InterpolationPickFirstStep(double initValue) { - super(initValue); - } - - public double getFirstStep(LineSearchMethod ls){ - if(ls.getPreviousStepUsed() != -1 && ls.getPreviousInitialGradient()!=0){ - double newStep = Math.min(300, 1.02*ls.getPreviousInitialGradient()*ls.getPreviousStepUsed()/ls.getInitialGradient()); - // System.out.println("proposing " + newStep); - return newStep; - - } - return _initValue; - } - public void collectInitValues(WolfRuleLineSearch ls){ - - } - - public void collectFinalValues(WolfRuleLineSearch ls){ - - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/LineSearchMethod.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/LineSearchMethod.java deleted file mode 100644 index 80cd7f39..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/LineSearchMethod.java +++ /dev/null @@ -1,14 +0,0 @@ -package optimization.linesearch; - - -public interface LineSearchMethod { - - double getStepSize(DifferentiableLineSearchObjective o); - - public double getInitialGradient(); - public double getPreviousInitialGradient(); - public double getPreviousStepUsed(); - - public void setInitialStep(double initial); - public void reset(); -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/NonNewtonInterpolationPickFirstStep.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/NonNewtonInterpolationPickFirstStep.java deleted file mode 100644 index 4b354fd9..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/NonNewtonInterpolationPickFirstStep.java +++ /dev/null @@ -1,33 +0,0 @@ -package optimization.linesearch; - -/** - * Non newtwon since we don't always try 1... - * Not sure if that is even usefull for newton - * @author javg - * - */ -public class NonNewtonInterpolationPickFirstStep extends GenericPickFirstStep{ - public NonNewtonInterpolationPickFirstStep(double initValue) { - super(initValue); - } - - public double getFirstStep(LineSearchMethod ls){ -// System.out.println("Previous step used " + ls.getPreviousStepUsed()); -// System.out.println("PreviousGradinebt " + ls.getPreviousInitialGradient()); -// System.out.println("CurrentGradinebt " + ls.getInitialGradient()); - if(ls.getPreviousStepUsed() != -1 && ls.getPreviousInitialGradient()!=0){ - double newStep = 1.01*ls.getPreviousInitialGradient()*ls.getPreviousStepUsed()/ls.getInitialGradient(); - //System.out.println("Suggesting " + newStep); - return newStep; - - } - return _initValue; - } - public void collectInitValues(WolfRuleLineSearch ls){ - - } - - public void collectFinalValues(WolfRuleLineSearch ls){ - - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/ProjectedDifferentiableLineSearchObjective.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/ProjectedDifferentiableLineSearchObjective.java deleted file mode 100644 index 29ccbc32..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/ProjectedDifferentiableLineSearchObjective.java +++ /dev/null @@ -1,137 +0,0 @@ -package optimization.linesearch; - -import optimization.gradientBasedMethods.Objective; -import optimization.gradientBasedMethods.ProjectedObjective; -import optimization.util.MathUtils; -import optimization.util.MatrixOutput; - - -/** - * See ArmijoLineSearchMinimizationAlongProjectionArc for description - * @author javg - * - */ -public class ProjectedDifferentiableLineSearchObjective extends DifferentiableLineSearchObjective{ - - - - ProjectedObjective obj; - public ProjectedDifferentiableLineSearchObjective(Objective o) { - super(o); - if(!(o instanceof ProjectedObjective)){ - System.out.println("Must receive a projected objective"); - throw new RuntimeException(); - } - obj = (ProjectedObjective) o; - } - - - - public double[] projectPoint (double[] point){ - return ((ProjectedObjective)o).projectPoint(point); - } - public void updateAlpha(double alpha){ - if(alpha < 0){ - System.out.println("alpha may not be smaller that zero"); - throw new RuntimeException(); - } - - if(obj.auxParameters == null){ - obj.auxParameters = new double[obj.getParameters().length]; - } - - nrIterations++; - - steps.add(alpha); - System.arraycopy(originalParameters, 0, obj.auxParameters, 0, obj.auxParameters.length); - - //Take a step into the search direction - -// MatrixOutput.printDoubleArray(obj.getGradient(), "gradient"); - -// alpha=gradients.get(0)*alpha/(gradients.get(gradients.size()-1)); - - //x_t+1 = x_t - alpha*gradient = x_t + alpha*direction - MathUtils.plusEquals(obj.auxParameters, searchDirection, alpha); -// MatrixOutput.printDoubleArray(obj.auxParameters, "before projection"); - obj.auxParameters = projectPoint(obj.auxParameters); -// MatrixOutput.printDoubleArray(obj.auxParameters, "after projection"); - o.setParameters(obj.auxParameters); -// System.out.println("new parameters"); -// o.printParameters(); - values.add(o.getValue()); - //Computes the new gradient x_k-[x_k-alpha*Gradient(x_k)]+ - MathUtils.minusEqualsInverse(originalParameters,obj.auxParameters,1); -// MatrixOutput.printDoubleArray(obj.auxParameters, "new gradient"); - //Dot product between the new direction and the new gradient - double gradient = MathUtils.dotProduct(obj.auxParameters,searchDirection); - gradients.add(gradient); - if(gradient > 0){ - System.out.println("Gradient on line search has to be smaller than zero"); - System.out.println("Iter: " + nrIterations); - MatrixOutput.printDoubleArray(obj.auxParameters, "new direction"); - MatrixOutput.printDoubleArray(searchDirection, "search direction"); - throw new RuntimeException(); - - } - - } - - /** - * - */ -// public void updateAlpha(double alpha){ -// -// if(alpha < 0){ -// System.out.println("alpha may not be smaller that zero"); -// throw new RuntimeException(); -// } -// -// nrIterations++; -// steps.add(alpha); -// //x_t+1 = x_t - alpha*direction -// System.arraycopy(originalParameters, 0, parametersChange, 0, parametersChange.length); -//// MatrixOutput.printDoubleArray(parametersChange, "parameters before step"); -//// System.out.println("Step" + alpha); -// MatrixOutput.printDoubleArray(originalGradient, "gradient + " + alpha); -// -// MathUtils.minusEquals(parametersChange, originalGradient, alpha); -// -// //Project the points into the feasibility set -//// MatrixOutput.printDoubleArray(parametersChange, "before projection"); -// //x_k(alpha) = [x_k - alpha*grad f(x_k)]+ -// parametersChange = projectPoint(parametersChange); -//// MatrixOutput.printDoubleArray(parametersChange, "after projection"); -// o.setParameters(parametersChange); -// values.add(o.getValue()); -// //Computes the new direction x_k-[x_k-alpha*Gradient(x_k)]+ -// -// direction=MathUtils.arrayMinus(parametersChange,originalParameters); -//// MatrixOutput.printDoubleArray(direction, "new direction"); -// -// double gradient = MathUtils.dotProduct(originalGradient,direction); -// gradients.add(gradient); -// if(gradient > 1E-10){ -// System.out.println("cosine " + gradient/(MathUtils.L2Norm(originalGradient)*MathUtils.L2Norm(direction))); -// -// -// System.out.println("not a descent direction for alpha " + alpha); -// System.arraycopy(originalParameters, 0, parametersChange, 0, parametersChange.length); -// MathUtils.minusEquals(parametersChange, originalGradient, 1E-20); -// -// parametersChange = projectPoint(parametersChange); -// direction=MathUtils.arrayMinus(parametersChange,originalParameters); -// gradient = MathUtils.dotProduct(originalGradient,direction); -// if(gradient > 0){ -// System.out.println("Direction is really non-descent evern for small alphas:" + gradient); -// } -// System.out.println("ProjecteLineSearchObjective: Should be a descent direction at " + nrIterations + ": "+ gradient); -//// System.out.println(Printing.doubleArrayToString(originalGradient, null,"Original gradient")); -//// System.out.println(Printing.doubleArrayToString(originalParameters, null,"Original parameters")); -//// System.out.println(Printing.doubleArrayToString(parametersChange, null,"Projected parameters")); -//// System.out.println(Printing.doubleArrayToString(direction, null,"Direction")); -// throw new RuntimeException(); -// } -// } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/WolfRuleLineSearch.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/WolfRuleLineSearch.java deleted file mode 100644 index 5489f2d0..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/WolfRuleLineSearch.java +++ /dev/null @@ -1,300 +0,0 @@ -package optimization.linesearch; - -import java.io.PrintStream; -import java.util.ArrayList; - -import optimization.util.Interpolation; - - - - -/** - * - * @author javg - * - */ -public class WolfRuleLineSearch implements LineSearchMethod{ - - GenericPickFirstStep pickFirstStep; - - double c1 = 1.0E-4; - double c2 = 0.9; - - //Application dependent - double maxStep=100; - - int extrapolationIteration; - int maxExtrapolationIteration = 1000; - - - double minZoomDiffTresh = 10E-10; - - - ArrayList<Double> steps; - ArrayList<Double> gradientDots; - ArrayList<Double> functionVals; - - int debugLevel = 0; - boolean foudStep = false; - - public WolfRuleLineSearch(GenericPickFirstStep pickFirstStep){ - this.pickFirstStep = pickFirstStep; - - } - - - - - public WolfRuleLineSearch(GenericPickFirstStep pickFirstStep, double c1, double c2){ - this.pickFirstStep = pickFirstStep; - initialStep = pickFirstStep.getFirstStep(this); - this.c1 = c1; - this.c2 = c2; - } - - public void setDebugLevel(int level){ - debugLevel = level; - } - - //Experiment - double previousStepPicked = -1;; - double previousInitGradientDot = -1; - double currentInitGradientDot = -1; - - double initialStep; - - - public void reset(){ - previousStepPicked = -1;; - previousInitGradientDot = -1; - currentInitGradientDot = -1; - if(steps != null) - steps.clear(); - if(gradientDots != null) - gradientDots.clear(); - if(functionVals != null) - functionVals.clear(); - } - - public void setInitialStep(double initial){ - initialStep = pickFirstStep.getFirstStep(this); - } - - - - /** - * Implements Wolf Line search as described in nocetal. - * This process consists in two stages. The first stage we try to satisfy the - * biggest step size that still satisfies the curvature condition. We keep increasing - * the initial step size until we find a step satisfying the curvature condition, we return - * success, we failed the sufficient increase so we cannot increase more and we can call zoom with - * that maximum step, or we pass the minimum in which case we can call zoom the same way. - * - */ - public double getStepSize(DifferentiableLineSearchObjective objective){ - //System.out.println("entering line search"); - - foudStep = false; - if(debugLevel >= 1){ - steps = new ArrayList<Double>(); - gradientDots = new ArrayList<Double>(); - functionVals =new ArrayList<Double>(); - } - - //test - currentInitGradientDot = objective.getInitialGradient(); - - - double previousValue = objective.getCurrentValue(); - double previousStep = 0; - double currentStep =pickFirstStep.getFirstStep(this); - for(extrapolationIteration = 0; - extrapolationIteration < maxExtrapolationIteration; extrapolationIteration++){ - - objective.updateAlpha(currentStep); - double currentValue = objective.getCurrentValue(); - if(debugLevel >= 1){ - steps.add(currentStep); - functionVals.add(currentValue); - gradientDots.add(objective.getCurrentGradient()); - } - - - //The current step does not satisfy the sufficient decrease condition anymore - // so we cannot get bigger than that calling zoom. - if(!WolfeConditions.suficientDecrease(objective,c1)|| - (extrapolationIteration > 0 && currentValue >= previousValue)){ - currentStep = zoom(objective,previousStep,currentStep,objective.nrIterations-1,objective.nrIterations); - break; - } - - //Satisfying both conditions ready to leave - if(WolfeConditions.sufficientCurvature(objective,c1,c2)){ - //Found step - foudStep = true; - break; - } - - /** - * This means that we passed the minimum already since the dot product that should be - * negative (descent direction) is now positive. So we cannot increase more. On the other hand - * since we know the direction is a descent direction the value the objective at the current step - * is for sure smaller than the preivous step so we change the order. - */ - if(objective.getCurrentGradient() >= 0){ - currentStep = zoom(objective,currentStep,previousStep,objective.nrIterations,objective.nrIterations-1); - break; - } - - - //Ok, so we can still get a bigger step, - double aux = currentStep; - //currentStep = currentStep*2; - if(Math.abs(currentStep-maxStep)>1.1e-2){ - currentStep = (currentStep+maxStep)/2; - }else{ - currentStep = currentStep*2; - } - previousStep = aux; - previousValue = currentValue; - //Could be done better - if(currentStep >= maxStep){ - System.out.println("Excedded max step...calling zoom with maxStepSize"); - currentStep = zoom(objective,previousStep,currentStep,objective.nrIterations-1,objective.nrIterations); - } - } - if(!foudStep){ - System.out.println("Wolfe Rule exceed number of iterations"); - if(debugLevel >= 1){ - printSmallWolfeStats(System.out); -// System.out.println("Line search values"); -// DebugHelpers.getLineSearchGraph(o, direction, originalParameters,origValue, origGradDirectionDot,c1,c2); - } - return -1; - } - if(debugLevel >= 1){ - printSmallWolfeStats(System.out); - } - - previousStepPicked = currentStep; - previousInitGradientDot = currentInitGradientDot; -// objective.printLineSearchSteps(); - return currentStep; - } - - - - - - public void printWolfeStats(PrintStream out){ - for(int i = 0; i < steps.size(); i++){ - out.println("Step " + steps.get(i) + " value " + functionVals.get(i) + " dot " + gradientDots.get(i)); - } - } - - public void printSmallWolfeStats(PrintStream out){ - for(int i = 0; i < steps.size(); i++){ - out.print(steps.get(i) + ":"+functionVals.get(i)+":"+gradientDots.get(i)+" "); - } - System.out.println(); - } - - - - /** - * Pick a step satisfying the strong wolfe condition from an given from lowerStep and higherStep - * picked on the routine above. - * - * Both lowerStep and higherStep have been evaluated, so we only need to pass the iteration where they have - * been evaluated and save extra evaluations. - * - * We know that lowerStepValue as to be smaller than higherStepValue, and that a point - * satisfying both conditions exists in such interval. - * - * LowerStep always satisfies at least the sufficient decrease - * @return - */ - public double zoom(DifferentiableLineSearchObjective o, double lowerStep, double higherStep, - int lowerStepIter, int higherStepIter){ - - if(debugLevel >=2){ - System.out.println("Entering zoom with " + lowerStep+"-"+higherStep); - } - - double currentStep=-1; - - int zoomIter = 0; - while(zoomIter < 1000){ - if(Math.abs(lowerStep-higherStep) < minZoomDiffTresh){ - o.updateAlpha(lowerStep); - if(debugLevel >= 1){ - steps.add(lowerStep); - functionVals.add(o.getCurrentValue()); - gradientDots.add(o.getCurrentGradient()); - } - foudStep = true; - return lowerStep; - } - - //Cubic interpolation - currentStep = - Interpolation.cubicInterpolation(lowerStep, o.getValue(lowerStepIter), o.getGradient(lowerStepIter), - higherStep, o.getValue(higherStepIter), o.getGradient(higherStepIter)); - - //Safeguard.... should not be required check in what condtions it is required - if(currentStep < 0 ){ - currentStep = (lowerStep+higherStep)/2; - } - if(Double.isNaN(currentStep) || Double.isInfinite(currentStep)){ - currentStep = (lowerStep+higherStep)/2; - } -// currentStep = (lowerStep+higherStep)/2; -// System.out.println("Trying "+currentStep); - o.updateAlpha(currentStep); - if(debugLevel >=1){ - steps.add(currentStep); - functionVals.add(o.getCurrentValue()); - gradientDots.add(o.getCurrentGradient()); - } - if(!WolfeConditions.suficientDecrease(o,c1) - || o.getCurrentValue() >= o.getValue(lowerStepIter)){ - higherStepIter = o.nrIterations; - higherStep = currentStep; - } - //Note when entering here the new step satisfies the sufficent decrease and - // or as a function value that is better than the previous best (lowerStepFunctionValues) - // so we either leave or change the value of the alpha low. - else{ - if(WolfeConditions.sufficientCurvature(o,c1,c2)){ - //Satisfies the both wolf conditions - foudStep = true; - break; - } - //If does not satisfy curvature - if(o.getCurrentGradient()*(higherStep-lowerStep) >= 0){ - higherStep = lowerStep; - higherStepIter = lowerStepIter; - } - lowerStep = currentStep; - lowerStepIter = o.nrIterations; - } - zoomIter++; - } - return currentStep; - } - - public double getInitialGradient() { - return currentInitGradientDot; - - } - - public double getPreviousInitialGradient() { - return previousInitGradientDot; - } - - public double getPreviousStepUsed() { - return previousStepPicked; - } - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/linesearch/WolfeConditions.java b/gi/posterior-regularisation/prjava/src/optimization/linesearch/WolfeConditions.java deleted file mode 100644 index dcc704eb..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/linesearch/WolfeConditions.java +++ /dev/null @@ -1,45 +0,0 @@ -package optimization.linesearch; - - -public class WolfeConditions { - - /** - * Sufficient Increase number. Default constant - */ - - - /** - * Value for suficient curvature: - * 0.9 - For newton and quase netwon methods - * 0.1 - Non linear conhugate gradient - */ - - int debugLevel = 0; - public void setDebugLevel(int level){ - debugLevel = level; - } - - public static boolean suficientDecrease(DifferentiableLineSearchObjective o, double c1){ - double value = o.getOriginalValue()+c1*o.getAlpha()*o.getInitialGradient(); -// System.out.println("Sufficient Decrease original "+value+" new "+ o.getCurrentValue()); - return o.getCurrentValue() <= value; - } - - - - - public static boolean sufficientCurvature(DifferentiableLineSearchObjective o, double c1, double c2){ -// if(debugLevel >= 2){ -// double current = Math.abs(o.getCurrentGradient()); -// double orig = -c2*o.getInitialGradient(); -// if(current <= orig){ -// return true; -// }else{ -// System.out.println("Not satistfying curvature condition curvature " + current + " wants " + orig); -// return false; -// } -// } - return Math.abs(o.getCurrentGradient()) <= -c2*o.getInitialGradient(); - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/projections/BoundsProjection.java b/gi/posterior-regularisation/prjava/src/optimization/projections/BoundsProjection.java deleted file mode 100644 index 0429d531..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/projections/BoundsProjection.java +++ /dev/null @@ -1,104 +0,0 @@ -package optimization.projections; - - -import java.util.Random; - -import optimization.util.MathUtils; -import optimization.util.MatrixOutput; - -/** - * Implements a projection into a box set defined by a and b. - * If either a or b are infinity then that bound is ignored. - * @author javg - * - */ -public class BoundsProjection extends Projection{ - - double a,b; - boolean ignoreA = false; - boolean ignoreB = false; - public BoundsProjection(double lowerBound, double upperBound) { - if(Double.isInfinite(lowerBound)){ - this.ignoreA = true; - }else{ - this.a =lowerBound; - } - if(Double.isInfinite(upperBound)){ - this.ignoreB = true; - }else{ - this.b =upperBound; - } - } - - - - /** - * Projects into the bounds - * a <= x_i <=b - */ - public void project(double[] original){ - for (int i = 0; i < original.length; i++) { - if(!ignoreA && original[i] < a){ - original[i] = a; - }else if(!ignoreB && original[i]>b){ - original[i]=b; - } - } - } - - /** - * Generates a random number between a and b. - */ - - Random r = new Random(); - - public double[] samplePoint(int numParams) { - double[] point = new double[numParams]; - for (int i = 0; i < point.length; i++) { - double rand = r.nextDouble(); - if(ignoreA && ignoreB){ - //Use const to avoid number near overflow - point[i] = rand*(1.E100+1.E100)-1.E100; - }else if(ignoreA){ - point[i] = rand*(b-1.E100)-1.E100; - }else if(ignoreB){ - point[i] = rand*(1.E100-a)-a; - }else{ - point[i] = rand*(b-a)-a; - } - } - return point; - } - - public static void main(String[] args) { - BoundsProjection sp = new BoundsProjection(0,Double.POSITIVE_INFINITY); - - - MatrixOutput.printDoubleArray(sp.samplePoint(3), "random 1"); - MatrixOutput.printDoubleArray(sp.samplePoint(3), "random 2"); - MatrixOutput.printDoubleArray(sp.samplePoint(3), "random 3"); - - double[] d = {-1.1,1.2,1.4}; - double[] original = d.clone(); - MatrixOutput.printDoubleArray(d, "before"); - - sp.project(d); - MatrixOutput.printDoubleArray(d, "after"); - System.out.println("Test projection: " + sp.testProjection(original, d)); - } - - double epsilon = 1.E-10; - public double[] perturbePoint(double[] point, int parameter){ - double[] newPoint = point.clone(); - if(!ignoreA && MathUtils.almost(point[parameter], a)){ - newPoint[parameter]+=epsilon; - }else if(!ignoreB && MathUtils.almost(point[parameter], b)){ - newPoint[parameter]-=epsilon; - }else{ - newPoint[parameter]-=epsilon; - } - return newPoint; - } - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/projections/Projection.java b/gi/posterior-regularisation/prjava/src/optimization/projections/Projection.java deleted file mode 100644 index b5a9f92f..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/projections/Projection.java +++ /dev/null @@ -1,72 +0,0 @@ -package optimization.projections; - -import optimization.util.MathUtils; -import optimization.util.MatrixOutput; -import util.ArrayMath; -import util.Printing; - - - -public abstract class Projection { - - - public abstract void project(double[] original); - - - /** - * From the projection theorem "Non-Linear Programming" page - * 201 fact 2. - * - * Given some z in R, and a vector x* in X; - * x* = z+ iif for all x in X - * (z-x*)'(x-x*) <= 0 where 0 is when x*=x - * See figure 2.16 in book - * - * @param original - * @param projected - * @return - */ - public boolean testProjection(double[] original, double[] projected){ - double[] original1 = original.clone(); - //System.out.println(Printing.doubleArrayToString(original1, null, "original")); - //System.out.println(Printing.doubleArrayToString(projected, null, "projected")); - MathUtils.minusEquals(original1, projected, 1); - //System.out.println(Printing.doubleArrayToString(original1, null, "minus1")); - for(int i = 0; i < 10; i++){ - double[] x = samplePoint(original.length); - // System.out.println(Printing.doubleArrayToString(x, null, "sample")); - //If the same this returns zero so we are there. - MathUtils.minusEquals(x, projected, 1); - // System.out.println(Printing.doubleArrayToString(x, null, "minus2")); - double dotProd = MathUtils.dotProduct(original1, x); - - // System.out.println("dot " + dotProd); - if(dotProd > 0) return false; - } - - //Perturbs the point a bit in all possible directions - for(int i = 0; i < original.length; i++){ - double[] x = perturbePoint(projected,i); - // System.out.println(Printing.doubleArrayToString(x, null, "perturbed")); - //If the same this returns zero so we are there. - MathUtils.minusEquals(x, projected, 1); - // System.out.println(Printing.doubleArrayToString(x, null, "minus2")); - double dotProd = MathUtils.dotProduct(original1, x); - - // System.out.println("dot " + dotProd); - if(dotProd > 0) return false; - } - - - - return true; - } - - //Samples a point from the constrained set - public abstract double[] samplePoint(int dimensions); - - //Perturbs a point a bit still leaving it at the constraints set - public abstract double[] perturbePoint(double[] point, int parameter); - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/projections/SimplexProjection.java b/gi/posterior-regularisation/prjava/src/optimization/projections/SimplexProjection.java deleted file mode 100644 index f22afcaf..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/projections/SimplexProjection.java +++ /dev/null @@ -1,127 +0,0 @@ -package optimization.projections; - - - -import java.util.Random; - -import optimization.util.MathUtils; -import optimization.util.MatrixOutput; - -public class SimplexProjection extends Projection{ - - double scale; - public SimplexProjection(double scale) { - this.scale = scale; - } - - /** - * projects the numbers of the array - * into a simplex of size. - * We follow the description of the paper - * "Efficient Projetions onto the l1-Ball - * for learning in high dimensions" - */ - public void project(double[] original){ - double[] ds = new double[original.length]; - System.arraycopy(original, 0, ds, 0, ds.length); - //If sum is smaller then zero then its ok - for (int i = 0; i < ds.length; i++) ds[i] = ds[i]>0? ds[i]:0; - double sum = MathUtils.sum(ds); - if (scale - sum >= -1.E-10 ){ - System.arraycopy(ds, 0, original, 0, ds.length); - //System.out.println("Not projecting"); - return; - } - //System.out.println("projecting " + sum + " scontraints " + scale); - util.Array.sortDescending(ds); - double currentSum = 0; - double previousTheta = 0; - double theta = 0; - for (int i = 0; i < ds.length; i++) { - currentSum+=ds[i]; - theta = (currentSum-scale)/(i+1); - if(ds[i]-theta < -1e-10){ - break; - } - previousTheta = theta; - } - //DEBUG - if(previousTheta < 0){ - System.out.println("Simple Projection: Theta is smaller than zero: " + previousTheta); - System.exit(-1); - } - for (int i = 0; i < original.length; i++) { - original[i] = Math.max(original[i]-previousTheta, 0); - } - } - - - - - - - /** - * Samples a point from the simplex of scale. Just sample - * random number from 0-scale and then if - * their sum is bigger then sum make them normalize. - * This is probably not sampling uniformly from the simplex but it is - * enough for our goals in here. - */ - Random r = new Random(); - public double[] samplePoint(int dimensions) { - double[] newPoint = new double[dimensions]; - double sum =0; - for (int i = 0; i < newPoint.length; i++) { - double rand = r.nextDouble()*scale; - sum+=rand; - newPoint[i]=rand; - } - //Normalize - if(sum > scale){ - for (int i = 0; i < newPoint.length; i++) { - newPoint[i]=scale*newPoint[i]/sum; - } - } - return newPoint; - } - - public static void main(String[] args) { - SimplexProjection sp = new SimplexProjection(1); - - - double[] point = sp.samplePoint(3); - MatrixOutput.printDoubleArray(point , "random 1 sum:" + MathUtils.sum(point)); - point = sp.samplePoint(3); - MatrixOutput.printDoubleArray(point , "random 2 sum:" + MathUtils.sum(point)); - point = sp.samplePoint(3); - MatrixOutput.printDoubleArray(point , "random 3 sum:" + MathUtils.sum(point)); - - double[] d = {0,1.1,-10}; - double[] original = d.clone(); - MatrixOutput.printDoubleArray(d, "before"); - - sp.project(d); - MatrixOutput.printDoubleArray(d, "after"); - System.out.println("Test projection: " + sp.testProjection(original, d)); - - } - - - double epsilon = 1.E-10; - public double[] perturbePoint(double[] point, int parameter){ - double[] newPoint = point.clone(); - if(MathUtils.almost(MathUtils.sum(point), scale)){ - newPoint[parameter]-=epsilon; - } - else if(point[parameter]==0){ - newPoint[parameter]+=epsilon; - }else if(MathUtils.almost(point[parameter], scale)){ - newPoint[parameter]-=epsilon; - } - else{ - newPoint[parameter]-=epsilon; - } - return newPoint; - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/CompositeStopingCriteria.java b/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/CompositeStopingCriteria.java deleted file mode 100644 index 15760f18..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/CompositeStopingCriteria.java +++ /dev/null @@ -1,33 +0,0 @@ -package optimization.stopCriteria; - -import java.util.ArrayList; - -import optimization.gradientBasedMethods.Objective; - -public class CompositeStopingCriteria implements StopingCriteria { - - ArrayList<StopingCriteria> criterias; - - public CompositeStopingCriteria() { - criterias = new ArrayList<StopingCriteria>(); - } - - public void add(StopingCriteria criteria){ - criterias.add(criteria); - } - - public boolean stopOptimization(Objective obj){ - for(StopingCriteria criteria: criterias){ - if(criteria.stopOptimization(obj)){ - return true; - } - } - return false; - } - - public void reset(){ - for(StopingCriteria criteria: criterias){ - criteria.reset(); - } - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/GradientL2Norm.java b/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/GradientL2Norm.java deleted file mode 100644 index 534ff833..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/GradientL2Norm.java +++ /dev/null @@ -1,30 +0,0 @@ -package optimization.stopCriteria; - -import optimization.gradientBasedMethods.Objective; -import optimization.util.MathUtils; - -public class GradientL2Norm implements StopingCriteria{ - - /** - * Stop if gradientNorm/(originalGradientNorm) smaller - * than gradientConvergenceValue - */ - protected double gradientConvergenceValue; - - - public GradientL2Norm(double gradientConvergenceValue){ - this.gradientConvergenceValue = gradientConvergenceValue; - } - - public void reset(){} - - public boolean stopOptimization(Objective obj){ - double norm = MathUtils.L2Norm(obj.gradient); - if(norm < gradientConvergenceValue){ - System.out.println("Gradient norm below treshold"); - return true; - } - return false; - - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedGradientL2Norm.java b/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedGradientL2Norm.java deleted file mode 100644 index 4a489641..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedGradientL2Norm.java +++ /dev/null @@ -1,48 +0,0 @@ -package optimization.stopCriteria; - -import optimization.gradientBasedMethods.Objective; -import optimization.gradientBasedMethods.ProjectedObjective; -import optimization.util.MathUtils; - -/** - * Divides the norm by the norm at the begining of the iteration - * @author javg - * - */ -public class NormalizedGradientL2Norm extends GradientL2Norm{ - - /** - * Stop if gradientNorm/(originalGradientNorm) smaller - * than gradientConvergenceValue - */ - protected double originalGradientNorm = -1; - - public void reset(){ - originalGradientNorm = -1; - } - public NormalizedGradientL2Norm(double gradientConvergenceValue){ - super(gradientConvergenceValue); - } - - - - - public boolean stopOptimization(Objective obj){ - double norm = MathUtils.L2Norm(obj.gradient); - if(originalGradientNorm == -1){ - originalGradientNorm = norm; - } - if(originalGradientNorm < 1E-10){ - System.out.println("Gradient norm is zero " + originalGradientNorm); - return true; - } - double normalizedNorm = 1.0*norm/originalGradientNorm; - if( normalizedNorm < gradientConvergenceValue){ - System.out.println("Gradient norm below normalized normtreshold: " + norm + " original: " + originalGradientNorm + " normalized norm: " + normalizedNorm); - return true; - }else{ -// System.out.println("projected gradient norm: " + norm); - return false; - } - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedProjectedGradientL2Norm.java b/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedProjectedGradientL2Norm.java deleted file mode 100644 index 5ae554c2..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedProjectedGradientL2Norm.java +++ /dev/null @@ -1,60 +0,0 @@ -package optimization.stopCriteria; - -import optimization.gradientBasedMethods.Objective; -import optimization.gradientBasedMethods.ProjectedObjective; -import optimization.util.MathUtils; - -/** - * Divides the norm by the norm at the begining of the iteration - * @author javg - * - */ -public class NormalizedProjectedGradientL2Norm extends ProjectedGradientL2Norm{ - - /** - * Stop if gradientNorm/(originalGradientNorm) smaller - * than gradientConvergenceValue - */ - double originalProjectedNorm = -1; - - public NormalizedProjectedGradientL2Norm(double gradientConvergenceValue){ - super(gradientConvergenceValue); - } - - public void reset(){ - originalProjectedNorm = -1; - } - - - double[] projectGradient(ProjectedObjective obj){ - - if(obj.auxParameters == null){ - obj.auxParameters = new double[obj.getNumParameters()]; - } - System.arraycopy(obj.getParameters(), 0, obj.auxParameters, 0, obj.getNumParameters()); - MathUtils.minusEquals(obj.auxParameters, obj.gradient, 1); - obj.auxParameters = obj.projectPoint(obj.auxParameters); - MathUtils.minusEquals(obj.auxParameters,obj.getParameters(),1); - return obj.auxParameters; - } - - public boolean stopOptimization(Objective obj){ - if(obj instanceof ProjectedObjective) { - ProjectedObjective o = (ProjectedObjective) obj; - double norm = MathUtils.L2Norm(projectGradient(o)); - if(originalProjectedNorm == -1){ - originalProjectedNorm = norm; - } - double normalizedNorm = 1.0*norm/originalProjectedNorm; - if( normalizedNorm < gradientConvergenceValue){ - System.out.println("Gradient norm below normalized normtreshold: " + norm + " original: " + originalProjectedNorm + " normalized norm: " + normalizedNorm); - return true; - }else{ -// System.out.println("projected gradient norm: " + norm); - return false; - } - } - System.out.println("Not a projected objective"); - throw new RuntimeException(); - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedValueDifference.java b/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedValueDifference.java deleted file mode 100644 index 6dbbc50d..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/NormalizedValueDifference.java +++ /dev/null @@ -1,54 +0,0 @@ -package optimization.stopCriteria; - -import optimization.gradientBasedMethods.Objective; -import optimization.util.MathUtils; - -public class NormalizedValueDifference implements StopingCriteria{ - - /** - * Stop if the different between values is smaller than a treshold - */ - protected double valueConvergenceValue=0.01; - protected double previousValue = Double.NaN; - protected double currentValue = Double.NaN; - - public NormalizedValueDifference(double valueConvergenceValue){ - this.valueConvergenceValue = valueConvergenceValue; - } - - public void reset(){ - previousValue = Double.NaN; - currentValue = Double.NaN; - } - - - public boolean stopOptimization(Objective obj){ - if(Double.isNaN(currentValue)){ - currentValue = obj.getValue(); - return false; - }else { - previousValue = currentValue; - currentValue = obj.getValue(); - if(previousValue != 0){ - double valueDiff = Math.abs(previousValue - currentValue)/Math.abs(previousValue); - if( valueDiff < valueConvergenceValue){ - System.out.println("Leaving different in values is to small: Prev " - + (previousValue/previousValue) + " Curr: " + (currentValue/previousValue) - + " diff: " + valueDiff); - return true; - } - }else{ - double valueDiff = Math.abs(previousValue - currentValue); - if( valueDiff < valueConvergenceValue){ - System.out.println("Leaving different in values is to small: Prev " - + (previousValue) + " Curr: " + (currentValue) - + " diff: " + valueDiff); - return true; - } - } - - return false; - } - - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/ProjectedGradientL2Norm.java b/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/ProjectedGradientL2Norm.java deleted file mode 100644 index aadf1fd5..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/ProjectedGradientL2Norm.java +++ /dev/null @@ -1,51 +0,0 @@ -package optimization.stopCriteria; - -import optimization.gradientBasedMethods.Objective; -import optimization.gradientBasedMethods.ProjectedObjective; -import optimization.util.MathUtils; - -public class ProjectedGradientL2Norm implements StopingCriteria{ - - /** - * Stop if gradientNorm/(originalGradientNorm) smaller - * than gradientConvergenceValue - */ - protected double gradientConvergenceValue; - - - public ProjectedGradientL2Norm(double gradientConvergenceValue){ - this.gradientConvergenceValue = gradientConvergenceValue; - } - - public void reset(){ - - } - - double[] projectGradient(ProjectedObjective obj){ - - if(obj.auxParameters == null){ - obj.auxParameters = new double[obj.getNumParameters()]; - } - System.arraycopy(obj.getParameters(), 0, obj.auxParameters, 0, obj.getNumParameters()); - MathUtils.minusEquals(obj.auxParameters, obj.gradient, 1); - obj.auxParameters = obj.projectPoint(obj.auxParameters); - MathUtils.minusEquals(obj.auxParameters,obj.getParameters(),1); - return obj.auxParameters; - } - - public boolean stopOptimization(Objective obj){ - if(obj instanceof ProjectedObjective) { - ProjectedObjective o = (ProjectedObjective) obj; - double norm = MathUtils.L2Norm(projectGradient(o)); - if(norm < gradientConvergenceValue){ - // System.out.println("Gradient norm below treshold: " + norm); - return true; - }else{ -// System.out.println("projected gradient norm: " + norm); - return false; - } - } - System.out.println("Not a projected objective"); - throw new RuntimeException(); - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/StopingCriteria.java b/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/StopingCriteria.java deleted file mode 100644 index 10cf0522..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/StopingCriteria.java +++ /dev/null @@ -1,8 +0,0 @@ -package optimization.stopCriteria; - -import optimization.gradientBasedMethods.Objective; - -public interface StopingCriteria { - public boolean stopOptimization(Objective obj); - public void reset(); -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/ValueDifference.java b/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/ValueDifference.java deleted file mode 100644 index e5d07229..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/stopCriteria/ValueDifference.java +++ /dev/null @@ -1,41 +0,0 @@ -package optimization.stopCriteria; - -import optimization.gradientBasedMethods.Objective; -import optimization.util.MathUtils; - -public class ValueDifference implements StopingCriteria{ - - /** - * Stop if the different between values is smaller than a treshold - */ - protected double valueConvergenceValue=0.01; - protected double previousValue = Double.NaN; - protected double currentValue = Double.NaN; - - public ValueDifference(double valueConvergenceValue){ - this.valueConvergenceValue = valueConvergenceValue; - } - - public void reset(){ - previousValue = Double.NaN; - currentValue = Double.NaN; - } - - public boolean stopOptimization(Objective obj){ - if(Double.isNaN(currentValue)){ - currentValue = obj.getValue(); - return false; - }else { - previousValue = currentValue; - currentValue = obj.getValue(); - if(previousValue - currentValue < valueConvergenceValue){ -// System.out.println("Leaving different in values is to small: Prev " -// + previousValue + " Curr: " + currentValue -// + " diff: " + (previousValue - currentValue)); - return true; - } - return false; - } - - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/util/Interpolation.java b/gi/posterior-regularisation/prjava/src/optimization/util/Interpolation.java deleted file mode 100644 index cdbdefc6..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/util/Interpolation.java +++ /dev/null @@ -1,37 +0,0 @@ -package optimization.util; - -public class Interpolation { - - /** - * Fits a cubic polinomyal to a function given two points, - * such that either gradB is bigger than zero or funcB >= funcA - * - * NonLinear Programming appendix C - * @param funcA - * @param gradA - * @param funcB - * @param gradB - */ - public final static double cubicInterpolation(double a, - double funcA, double gradA, double b,double funcB, double gradB ){ - if(gradB < 0 && funcA > funcB){ - System.out.println("Cannot call cubic interpolation"); - return -1; - } - - double z = 3*(funcA-funcB)/(b-a) + gradA + gradB; - double w = Math.sqrt(z*z - gradA*gradB); - double min = b -(gradB+w-z)*(b-a)/(gradB-gradA+2*w); - return min; - } - - public final static double quadraticInterpolation(double initFValue, - double initGrad, double point,double pointFValue){ - double min = -1*initGrad*point*point/(2*(pointFValue-initGrad*point-initFValue)); - return min; - } - - public static void main(String[] args) { - - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/util/Logger.java b/gi/posterior-regularisation/prjava/src/optimization/util/Logger.java deleted file mode 100644 index 5343a39b..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/util/Logger.java +++ /dev/null @@ -1,7 +0,0 @@ -package optimization.util; - -public class Logger { - - - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/util/MathUtils.java b/gi/posterior-regularisation/prjava/src/optimization/util/MathUtils.java deleted file mode 100644 index af66f82c..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/util/MathUtils.java +++ /dev/null @@ -1,339 +0,0 @@ -package optimization.util; - -import java.util.Arrays; - - - -public class MathUtils { - - /** - * - * @param vector - * @return - */ - public static double L2Norm(double[] vector){ - double value = 0; - for(int i = 0; i < vector.length; i++){ - double v = vector[i]; - value+=v*v; - } - return Math.sqrt(value); - } - - public static double sum(double[] v){ - double sum = 0; - for (int i = 0; i < v.length; i++) { - sum+=v[i]; - } - return sum; - } - - - - - /** - * w = w + v - * @param w - * @param v - */ - public static void plusEquals(double[] w, double[] v) { - for(int i=0; i<w.length;i++){ - w[i] += w[i] + v[i]; - } - } - - /** - * w[i] = w[i] + v - * @param w - * @param v - */ - public static void plusEquals(double[] w, double v) { - for(int i=0; i<w.length;i++){ - w[i] += w[i] + v; - } - } - - /** - * w[i] = w[i] - v - * @param w - * @param v - */ - public static void minusEquals(double[] w, double v) { - for(int i=0; i<w.length;i++){ - w[i] -= w[i] + v; - } - } - - /** - * w = w + a*v - * @param w - * @param v - * @param a - */ - public static void plusEquals(double[] w, double[] v, double a) { - for(int i=0; i<w.length;i++){ - w[i] += a*v[i]; - } - } - - /** - * w = w - a*v - * @param w - * @param v - * @param a - */ - public static void minusEquals(double[] w, double[] v, double a) { - for(int i=0; i<w.length;i++){ - w[i] -= a*v[i]; - } - } - /** - * v = w - a*v - * @param w - * @param v - * @param a - */ - public static void minusEqualsInverse(double[] w, double[] v, double a) { - for(int i=0; i<w.length;i++){ - v[i] = w[i] - a*v[i]; - } - } - - public static double dotProduct(double[] w, double[] v){ - double accum = 0; - for(int i=0; i<w.length;i++){ - accum += w[i]*v[i]; - } - return accum; - } - - public static double[] arrayMinus(double[]w, double[]v){ - double result[] = w.clone(); - for(int i=0; i<w.length;i++){ - result[i] -= v[i]; - } - return result; - } - - public static double[] arrayMinus(double[] result , double[]w, double[]v){ - for(int i=0; i<w.length;i++){ - result[i] = w[i]-v[i]; - } - return result; - } - - public static double[] negation(double[]w){ - double result[] = new double[w.length]; - for(int i=0; i<w.length;i++){ - result[i] = -w[i]; - } - return result; - } - - public static double square(double value){ - return value*value; - } - public static double[][] outerProduct(double[] w, double[] v){ - double[][] result = new double[w.length][v.length]; - for(int i = 0; i < w.length; i++){ - for(int j = 0; j < v.length; j++){ - result[i][j] = w[i]*v[j]; - } - } - return result; - } - /** - * results = a*W*V - * @param w - * @param v - * @param a - * @return - */ - public static double[][] weightedouterProduct(double[] w, double[] v, double a){ - double[][] result = new double[w.length][v.length]; - for(int i = 0; i < w.length; i++){ - for(int j = 0; j < v.length; j++){ - result[i][j] = a*w[i]*v[j]; - } - } - return result; - } - - public static double[][] identity(int size){ - double[][] result = new double[size][size]; - for(int i = 0; i < size; i++){ - result[i][i] = 1; - } - return result; - } - - /** - * v -= w - * @param v - * @param w - */ - public static void minusEquals(double[][] w, double[][] v){ - for(int i = 0; i < w.length; i++){ - for(int j = 0; j < w[0].length; j++){ - w[i][j] -= v[i][j]; - } - } - } - - /** - * v[i][j] -= a*w[i][j] - * @param v - * @param w - */ - public static void minusEquals(double[][] w, double[][] v, double a){ - for(int i = 0; i < w.length; i++){ - for(int j = 0; j < w[0].length; j++){ - w[i][j] -= a*v[i][j]; - } - } - } - - /** - * v += w - * @param v - * @param w - */ - public static void plusEquals(double[][] w, double[][] v){ - for(int i = 0; i < w.length; i++){ - for(int j = 0; j < w[0].length; j++){ - w[i][j] += v[i][j]; - } - } - } - - /** - * v[i][j] += a*w[i][j] - * @param v - * @param w - */ - public static void plusEquals(double[][] w, double[][] v, double a){ - for(int i = 0; i < w.length; i++){ - for(int j = 0; j < w[0].length; j++){ - w[i][j] += a*v[i][j]; - } - } - } - - - /** - * results = w*v - * @param w - * @param v - * @return - */ - public static double[][] matrixMultiplication(double[][] w,double[][] v){ - int w1 = w.length; - int w2 = w[0].length; - int v1 = v.length; - int v2 = v[0].length; - - if(w2 != v1){ - System.out.println("Matrix dimensions do not agree..."); - System.exit(-1); - } - - double[][] result = new double[w1][v2]; - for(int w_i1 = 0; w_i1 < w1; w_i1++){ - for(int v_i2 = 0; v_i2 < v2; v_i2++){ - double sum = 0; - for(int w_i2 = 0; w_i2 < w2; w_i2++){ - sum += w[w_i1 ][w_i2]*v[w_i2][v_i2]; - } - result[w_i1][v_i2] = sum; - } - } - return result; - } - - /** - * w = w.*v - * @param w - * @param v - */ - public static void matrixScalarMultiplication(double[][] w,double v){ - int w1 = w.length; - int w2 = w[0].length; - for(int w_i1 = 0; w_i1 < w1; w_i1++){ - for(int w_i2 = 0; w_i2 < w2; w_i2++){ - w[w_i1 ][w_i2] *= v; - } - } - } - - public static void scalarMultiplication(double[] w,double v){ - int w1 = w.length; - for(int w_i1 = 0; w_i1 < w1; w_i1++){ - w[w_i1 ] *= v; - } - - } - - public static double[] matrixVector(double[][] w,double[] v){ - int w1 = w.length; - int w2 = w[0].length; - int v1 = v.length; - - if(w2 != v1){ - System.out.println("Matrix dimensions do not agree..."); - System.exit(-1); - } - - double[] result = new double[w1]; - for(int w_i1 = 0; w_i1 < w1; w_i1++){ - double sum = 0; - for(int w_i2 = 0; w_i2 < w2; w_i2++){ - sum += w[w_i1 ][w_i2]*v[w_i2]; - } - result[w_i1] = sum; - } - return result; - } - - public static boolean allPositive(double[] array){ - for (int i = 0; i < array.length; i++) { - if(array[i] < 0) return false; - } - return true; - } - - - - - - public static void main(String[] args) { - double[][] m1 = new double[2][2]; - m1[0][0]=2; - m1[1][0]=2; - m1[0][1]=2; - m1[1][1]=2; - MatrixOutput.printDoubleArray(m1, "m1"); - double[][] m2 = new double[2][2]; - m2[0][0]=3; - m2[1][0]=3; - m2[0][1]=3; - m2[1][1]=3; - MatrixOutput.printDoubleArray(m2, "m2"); - double[][] result = matrixMultiplication(m1, m2); - MatrixOutput.printDoubleArray(result, "result"); - matrixScalarMultiplication(result, 3); - MatrixOutput.printDoubleArray(result, "result after multiply by 3"); - } - - public static boolean almost(double a, double b, double prec){ - return Math.abs(a-b)/Math.abs(a+b) <= prec || (almostZero(a) && almostZero(b)); - } - - public static boolean almost(double a, double b){ - return Math.abs(a-b)/Math.abs(a+b) <= 1e-10 || (almostZero(a) && almostZero(b)); - } - - public static boolean almostZero(double a) { - return Math.abs(a) <= 1e-30; - } - -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/util/MatrixOutput.java b/gi/posterior-regularisation/prjava/src/optimization/util/MatrixOutput.java deleted file mode 100644 index 9fbdf955..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/util/MatrixOutput.java +++ /dev/null @@ -1,28 +0,0 @@ -package optimization.util; - - -public class MatrixOutput { - public static void printDoubleArray(double[][] array, String arrayName) { - int size1 = array.length; - int size2 = array[0].length; - System.out.println(arrayName); - for (int i = 0; i < size1; i++) { - for (int j = 0; j < size2; j++) { - System.out.print(" " + StaticTools.prettyPrint(array[i][j], - "00.00E00", 4) + " "); - - } - System.out.println(); - } - System.out.println(); - } - - public static void printDoubleArray(double[] array, String arrayName) { - System.out.println(arrayName); - for (int i = 0; i < array.length; i++) { - System.out.print(" " + StaticTools.prettyPrint(array[i], - "00.00E00", 4) + " "); - } - System.out.println(); - } -} diff --git a/gi/posterior-regularisation/prjava/src/optimization/util/StaticTools.java b/gi/posterior-regularisation/prjava/src/optimization/util/StaticTools.java deleted file mode 100644 index bcabee06..00000000 --- a/gi/posterior-regularisation/prjava/src/optimization/util/StaticTools.java +++ /dev/null @@ -1,180 +0,0 @@ -package optimization.util; - - -import java.io.File; -import java.io.PrintStream; - -public class StaticTools { - - static java.text.DecimalFormat fmt = new java.text.DecimalFormat(); - - public static void createDir(String directory) { - - File dir = new File(directory); - if (!dir.isDirectory()) { - boolean success = dir.mkdirs(); - if (!success) { - System.out.println("Unable to create directory " + directory); - System.exit(0); - } - System.out.println("Created directory " + directory); - } else { - System.out.println("Reusing directory " + directory); - } - } - - /* - * q and p are indexed by source/foreign Sum_S(q) = 1 the same for p KL(q,p) = - * Eq*q/p - */ - public static double KLDistance(double[][] p, double[][] q, int sourceSize, - int foreignSize) { - double totalKL = 0; - // common.StaticTools.printMatrix(q, sourceSize, foreignSize, "q", - // System.out); - // common.StaticTools.printMatrix(p, sourceSize, foreignSize, "p", - // System.out); - for (int i = 0; i < sourceSize; i++) { - double kl = 0; - for (int j = 0; j < foreignSize; j++) { - assert !Double.isNaN(q[i][j]) : "KLDistance q: prob is NaN"; - assert !Double.isNaN(p[i][j]) : "KLDistance p: prob is NaN"; - if (p[i][j] == 0 || q[i][j] == 0) { - continue; - } else { - kl += q[i][j] * Math.log(q[i][j] / p[i][j]); - } - - } - totalKL += kl; - } - assert !Double.isNaN(totalKL) : "KLDistance: prob is NaN"; - if (totalKL < -1.0E-10) { - System.out.println("KL Smaller than zero " + totalKL); - System.out.println("Source Size" + sourceSize); - System.out.println("Foreign Size" + foreignSize); - StaticTools.printMatrix(q, sourceSize, foreignSize, "q", - System.out); - StaticTools.printMatrix(p, sourceSize, foreignSize, "p", - System.out); - System.exit(-1); - } - return totalKL / sourceSize; - } - - /* - * indexed the by [fi][si] - */ - public static double KLDistancePrime(double[][] p, double[][] q, - int sourceSize, int foreignSize) { - double totalKL = 0; - for (int i = 0; i < sourceSize; i++) { - double kl = 0; - for (int j = 0; j < foreignSize; j++) { - assert !Double.isNaN(q[j][i]) : "KLDistance q: prob is NaN"; - assert !Double.isNaN(p[j][i]) : "KLDistance p: prob is NaN"; - if (p[j][i] == 0 || q[j][i] == 0) { - continue; - } else { - kl += q[j][i] * Math.log(q[j][i] / p[j][i]); - } - - } - totalKL += kl; - } - assert !Double.isNaN(totalKL) : "KLDistance: prob is NaN"; - return totalKL / sourceSize; - } - - public static double Entropy(double[][] p, int sourceSize, int foreignSize) { - double totalE = 0; - for (int i = 0; i < foreignSize; i++) { - double e = 0; - for (int j = 0; j < sourceSize; j++) { - e += p[i][j] * Math.log(p[i][j]); - } - totalE += e; - } - return totalE / sourceSize; - } - - public static double[][] copyMatrix(double[][] original, int sourceSize, - int foreignSize) { - double[][] result = new double[sourceSize][foreignSize]; - for (int i = 0; i < sourceSize; i++) { - for (int j = 0; j < foreignSize; j++) { - result[i][j] = original[i][j]; - } - } - return result; - } - - public static void printMatrix(double[][] matrix, int sourceSize, - int foreignSize, String info, PrintStream out) { - - java.text.DecimalFormat fmt = new java.text.DecimalFormat(); - fmt.setMaximumFractionDigits(3); - fmt.setMaximumIntegerDigits(3); - fmt.setMinimumFractionDigits(3); - fmt.setMinimumIntegerDigits(3); - - out.println(info); - - for (int i = 0; i < foreignSize; i++) { - for (int j = 0; j < sourceSize; j++) { - out.print(prettyPrint(matrix[j][i], ".00E00", 6) + " "); - } - out.println(); - } - out.println(); - out.println(); - } - - public static void printMatrix(int[][] matrix, int sourceSize, - int foreignSize, String info, PrintStream out) { - - out.println(info); - for (int i = 0; i < foreignSize; i++) { - for (int j = 0; j < sourceSize; j++) { - out.print(matrix[j][i] + " "); - } - out.println(); - } - out.println(); - out.println(); - } - - public static String formatTime(long duration) { - StringBuilder sb = new StringBuilder(); - double d = duration / 1000; - fmt.applyPattern("00"); - sb.append(fmt.format((int) (d / (60 * 60))) + ":"); - d -= ((int) d / (60 * 60)) * 60 * 60; - sb.append(fmt.format((int) (d / 60)) + ":"); - d -= ((int) d / 60) * 60; - fmt.applyPattern("00.0"); - sb.append(fmt.format(d)); - return sb.toString(); - } - - public static String prettyPrint(double d, String patt, int len) { - fmt.applyPattern(patt); - String s = fmt.format(d); - while (s.length() < len) { - s = " " + s; - } - return s; - } - - - public static long getUsedMemory(){ - System.gc(); - return (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/ (1024 * 1024); - } - - public final static boolean compareDoubles(double d1, double d2){ - return Math.abs(d1-d2) <= 1.E-10; - } - - -} diff --git a/gi/posterior-regularisation/prjava/src/phrase/Agree.java b/gi/posterior-regularisation/prjava/src/phrase/Agree.java deleted file mode 100644 index 8f7b499e..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/Agree.java +++ /dev/null @@ -1,204 +0,0 @@ -package phrase;
-
-import gnu.trove.TIntArrayList;
-
-import io.FileUtil;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-
-import phrase.Corpus.Edge;
-
-public class Agree {
- PhraseCluster model1;
- C2F model2;
- Corpus c;
- private int K,n_phrases, n_words, n_contexts, n_positions1,n_positions2;
-
- /**@brief sum of loglikelihood of two
- * individual models
- */
- public double llh;
- /**@brief Bhattacharyya distance
- *
- */
- public double bdist;
- /**
- *
- * @param numCluster
- * @param corpus
- */
- public Agree(int numCluster, Corpus corpus){
-
- model1=new PhraseCluster(numCluster, corpus);
- model2=new C2F(numCluster,corpus);
- c=corpus;
- n_words=c.getNumWords();
- n_phrases=c.getNumPhrases();
- n_contexts=c.getNumContexts();
- n_positions1=c.getNumContextPositions();
- n_positions2=2;
- K=numCluster;
-
- }
-
- /**@brief test
- *
- */
- public static void main(String args[]){
- //String in="../pdata/canned.con";
- String in="../pdata/btec.con";
- String out="../pdata/posterior.out";
- int numCluster=25;
- Corpus corpus = null;
- File infile = new File(in);
- try {
- System.out.println("Reading concordance from " + infile);
- corpus = Corpus.readFromFile(FileUtil.reader(infile));
- corpus.printStats(System.out);
- } catch (IOException e) {
- System.err.println("Failed to open input file: " + infile);
- e.printStackTrace();
- System.exit(1);
- }
-
- Agree agree=new Agree(numCluster, corpus);
- int iter=20;
- for(int i=0;i<iter;i++){
- agree.EM();
- System.out.println("Iter"+i+", llh: "+agree.llh+
- ", divergence:"+agree.bdist+
- " sum: "+(agree.llh+agree.bdist));
- }
-
- File outfile = new File (out);
- try {
- PrintStream ps = FileUtil.printstream(outfile);
- agree.displayPosterior(ps);
- // ps.println();
- // c2f.displayModelParam(ps);
- ps.close();
- } catch (IOException e) {
- System.err.println("Failed to open output file: " + outfile);
- e.printStackTrace();
- System.exit(1);
- }
-
- }
-
- public double EM(){
-
- double [][][]exp_emit1=new double [K][n_positions1][n_words];
- double [][]exp_pi1=new double[n_phrases][K];
-
- double [][][]exp_emit2=new double [K][n_positions2][n_words];
- double [][]exp_pi2=new double[n_contexts][K];
-
- llh=0;
- bdist=0;
- //E
- for(int context=0; context< n_contexts; context++){
-
- List<Edge> contexts = c.getEdgesForContext(context);
-
- for (int ctx=0; ctx<contexts.size(); ctx++){
- Edge edge = contexts.get(ctx);
- int phrase=edge.getPhraseId();
- double p[]=posterior(edge);
- double z = arr.F.l1norm(p);
- assert z > 0;
- bdist += edge.getCount() * Math.log(z);
- arr.F.l1normalize(p);
-
- double count = edge.getCount();
- //increment expected count
- TIntArrayList phraseToks = edge.getPhrase();
- TIntArrayList contextToks = edge.getContext();
- for(int tag=0;tag<K;tag++){
-
- for(int position=0;position<n_positions1;position++){
- exp_emit1[tag][position][contextToks.get(position)]+=p[tag]*count;
- }
-
- exp_emit2[tag][0][phraseToks.get(0)]+=p[tag]*count;
- exp_emit2[tag][1][phraseToks.get(phraseToks.size()-1)]+=p[tag]*count;
-
- exp_pi1[phrase][tag]+=p[tag]*count;
- exp_pi2[context][tag]+=p[tag]*count;
- }
- }
- }
-
- //System.out.println("Log likelihood: "+loglikelihood);
-
- //M
- for(double [][]i:exp_emit1){
- for(double []j:i){
- arr.F.l1normalize(j);
- }
- }
-
- for(double []j:exp_pi1){
- arr.F.l1normalize(j);
- }
-
- for(double [][]i:exp_emit2){
- for(double []j:i){
- arr.F.l1normalize(j);
- }
- }
-
- for(double []j:exp_pi2){
- arr.F.l1normalize(j);
- }
-
- model1.emit=exp_emit1;
- model1.pi=exp_pi1;
- model2.emit=exp_emit2;
- model2.pi=exp_pi2;
-
- return llh;
- }
-
- public double[] posterior(Corpus.Edge edge)
- {
- double[] prob1=model1.posterior(edge);
- double[] prob2=model2.posterior(edge);
-
- llh+=edge.getCount()*Math.log(arr.F.l1norm(prob1));
- llh+=edge.getCount()*Math.log(arr.F.l1norm(prob2));
- arr.F.l1normalize(prob1);
- arr.F.l1normalize(prob2);
-
- for(int i=0;i<prob1.length;i++){
- prob1[i]*=prob2[i];
- prob1[i]=Math.sqrt(prob1[i]);
- }
-
- return prob1;
- }
-
- public void displayPosterior(PrintStream ps)
- {
- displayPosterior(ps, c.getEdges());
- }
-
- public void displayPosterior(PrintStream ps, List<Edge> test)
- {
- for (Edge edge : test)
- {
- double probs[] = posterior(edge);
- arr.F.l1normalize(probs);
-
- // emit phrase
- ps.print(edge.getPhraseString());
- ps.print("\t");
- ps.print(edge.getContextString(true));
- int t=arr.F.argmax(probs);
- ps.println(" ||| C=" + t);
- }
- }
-
-}
diff --git a/gi/posterior-regularisation/prjava/src/phrase/Agree2Sides.java b/gi/posterior-regularisation/prjava/src/phrase/Agree2Sides.java deleted file mode 100644 index 031f887f..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/Agree2Sides.java +++ /dev/null @@ -1,197 +0,0 @@ -package phrase;
-
-import gnu.trove.TIntArrayList;
-
-import io.FileUtil;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-
-import phrase.Corpus.Edge;
-
-public class Agree2Sides {
- PhraseCluster model1,model2;
- Corpus c1,c2;
- private int K;
-
- /**@brief sum of loglikelihood of two
- * individual models
- */
- public double llh;
- /**@brief Bhattacharyya distance
- *
- */
- public double bdist;
- /**
- *
- * @param numCluster
- * @param corpus
- */
- public Agree2Sides(int numCluster, Corpus corpus1 , Corpus corpus2 ){
-
- model1=new PhraseCluster(numCluster, corpus1);
- model2=new PhraseCluster(numCluster,corpus2);
- c1=corpus1;
- c2=corpus2;
- K=numCluster;
-
- }
-
- /**@brief test
- *
- */
- public static void main(String args[]){
- //String in="../pdata/canned.con";
- // String in="../pdata/btec.con";
- String in1="../pdata/source.txt";
- String in2="../pdata/target.txt";
- String out="../pdata/posterior.out";
- int numCluster=25;
- Corpus corpus1 = null,corpus2=null;
- File infile1 = new File(in1),infile2=new File(in2);
- try {
- System.out.println("Reading concordance from " + infile1);
- corpus1 = Corpus.readFromFile(FileUtil.reader(infile1));
- System.out.println("Reading concordance from " + infile2);
- corpus2 = Corpus.readFromFile(FileUtil.reader(infile2));
- corpus1.printStats(System.out);
- } catch (IOException e) {
- System.err.println("Failed to open input file: " + infile1);
- e.printStackTrace();
- System.exit(1);
- }
-
- Agree2Sides agree=new Agree2Sides(numCluster, corpus1,corpus2);
- int iter=20;
- for(int i=0;i<iter;i++){
- agree.EM();
- System.out.println("Iter"+i+", llh: "+agree.llh+
- ", divergence:"+agree.bdist+
- " sum: "+(agree.llh+agree.bdist));
- }
-
- File outfile = new File (out);
- try {
- PrintStream ps = FileUtil.printstream(outfile);
- agree.displayPosterior(ps);
- // ps.println();
- // c2f.displayModelParam(ps);
- ps.close();
- } catch (IOException e) {
- System.err.println("Failed to open output file: " + outfile);
- e.printStackTrace();
- System.exit(1);
- }
-
- }
-
- public double EM(){
-
- double [][][]exp_emit1=new double [K][c1.getNumContextPositions()][c1.getNumWords()];
- double [][]exp_pi1=new double[c1.getNumPhrases()][K];
-
- double [][][]exp_emit2=new double [K][c2.getNumContextPositions()][c2.getNumWords()];
- double [][]exp_pi2=new double[c2.getNumPhrases()][K];
-
- llh=0;
- bdist=0;
- //E
- for(int i=0;i<c1.getEdges().size();i++){
- Edge edge1=c1.getEdges().get(i);
- Edge edge2=c2.getEdges().get(i);
- double p[]=posterior(i);
- double z = arr.F.l1norm(p);
- assert z > 0;
- bdist += edge1.getCount() * Math.log(z);
- arr.F.l1normalize(p);
- double count = edge1.getCount();
- //increment expected count
- TIntArrayList contextToks1 = edge1.getContext();
- TIntArrayList contextToks2 = edge2.getContext();
- int phrase1=edge1.getPhraseId();
- int phrase2=edge2.getPhraseId();
- for(int tag=0;tag<K;tag++){
- for(int position=0;position<c1.getNumContextPositions();position++){
- exp_emit1[tag][position][contextToks1.get(position)]+=p[tag]*count;
- }
- for(int position=0;position<c2.getNumContextPositions();position++){
- exp_emit2[tag][position][contextToks2.get(position)]+=p[tag]*count;
- }
- exp_pi1[phrase1][tag]+=p[tag]*count;
- exp_pi2[phrase2][tag]+=p[tag]*count;
- }
- }
-
- //System.out.println("Log likelihood: "+loglikelihood);
-
- //M
- for(double [][]i:exp_emit1){
- for(double []j:i){
- arr.F.l1normalize(j);
- }
- }
-
- for(double []j:exp_pi1){
- arr.F.l1normalize(j);
- }
-
- for(double [][]i:exp_emit2){
- for(double []j:i){
- arr.F.l1normalize(j);
- }
- }
-
- for(double []j:exp_pi2){
- arr.F.l1normalize(j);
- }
-
- model1.emit=exp_emit1;
- model1.pi=exp_pi1;
- model2.emit=exp_emit2;
- model2.pi=exp_pi2;
-
- return llh;
- }
-
- public double[] posterior(int edgeIdx)
- {
- return posterior(c1.getEdges().get(edgeIdx), c2.getEdges().get(edgeIdx));
- }
-
- public double[] posterior(Edge e1, Edge e2)
- {
- double[] prob1=model1.posterior(e1);
- double[] prob2=model2.posterior(e2);
-
- llh+=e1.getCount()*Math.log(arr.F.l1norm(prob1));
- llh+=e2.getCount()*Math.log(arr.F.l1norm(prob2));
- arr.F.l1normalize(prob1);
- arr.F.l1normalize(prob2);
-
- for(int i=0;i<prob1.length;i++){
- prob1[i]*=prob2[i];
- prob1[i]=Math.sqrt(prob1[i]);
- }
-
- return prob1;
- }
-
- public void displayPosterior(PrintStream ps)
- {
- for (int i=0;i<c1.getEdges().size();i++)
- {
- Edge edge=c1.getEdges().get(i);
- double probs[] = posterior(i);
- arr.F.l1normalize(probs);
-
- // emit phrase
- ps.print(edge.getPhraseString());
- ps.print("\t");
- ps.print(edge.getContextString(true));
- int t=arr.F.argmax(probs);
- ps.println(" ||| C=" + t);
- }
- }
-}
diff --git a/gi/posterior-regularisation/prjava/src/phrase/C2F.java b/gi/posterior-regularisation/prjava/src/phrase/C2F.java deleted file mode 100644 index e8783950..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/C2F.java +++ /dev/null @@ -1,216 +0,0 @@ -package phrase;
-
-import gnu.trove.TIntArrayList;
-
-import io.FileUtil;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.Arrays;
-import java.util.List;
-
-import phrase.Corpus.Edge;
-
-/**
- * @brief context generates phrase
- * @author desaic
- *
- */
-public class C2F {
- public int K;
- private int n_words, n_contexts, n_positions;
- public Corpus c;
-
- /**@brief
- * emit[tag][position][word] = p(word | tag, position in phrase)
- */
- public double emit[][][];
- /**@brief
- * pi[context][tag] = p(tag | context)
- */
- public double pi[][];
-
- public C2F(int numCluster, Corpus corpus){
- K=numCluster;
- c=corpus;
- n_words=c.getNumWords();
- n_contexts=c.getNumContexts();
-
- //number of words in a phrase to be considered
- //currently the first and last word in source and target
- //if the phrase has length 1 in either dimension then
- //we use the same word for two positions
- n_positions=c.phraseEdges(c.getEdges().get(0).getPhrase()).size();
-
- emit=new double [K][n_positions][n_words];
- pi=new double[n_contexts][K];
-
- for(double [][]i:emit){
- for(double []j:i){
- arr.F.randomise(j);
- }
- }
-
- for(double []j:pi){
- arr.F.randomise(j);
- }
- }
-
- /**@brief test
- *
- */
- public static void main(String args[]){
- String in="../pdata/canned.con";
- String out="../pdata/posterior.out";
- int numCluster=25;
- Corpus corpus = null;
- File infile = new File(in);
- try {
- System.out.println("Reading concordance from " + infile);
- corpus = Corpus.readFromFile(FileUtil.reader(infile));
- corpus.printStats(System.out);
- } catch (IOException e) {
- System.err.println("Failed to open input file: " + infile);
- e.printStackTrace();
- System.exit(1);
- }
-
- C2F c2f=new C2F(numCluster,corpus);
- int iter=20;
- double llh=0;
- for(int i=0;i<iter;i++){
- llh=c2f.EM();
- System.out.println("Iter"+i+", llh: "+llh);
- }
-
- File outfile = new File (out);
- try {
- PrintStream ps = FileUtil.printstream(outfile);
- c2f.displayPosterior(ps);
- // ps.println();
- // c2f.displayModelParam(ps);
- ps.close();
- } catch (IOException e) {
- System.err.println("Failed to open output file: " + outfile);
- e.printStackTrace();
- System.exit(1);
- }
-
- }
-
- public double EM(){
- double [][][]exp_emit=new double [K][n_positions][n_words];
- double [][]exp_pi=new double[n_contexts][K];
-
- double loglikelihood=0;
-
- //E
- for(int context=0; context< n_contexts; context++){
-
- List<Edge> contexts = c.getEdgesForContext(context);
-
- for (int ctx=0; ctx<contexts.size(); ctx++){
- Edge edge = contexts.get(ctx);
- double p[]=posterior(edge);
- double z = arr.F.l1norm(p);
- assert z > 0;
- loglikelihood += edge.getCount() * Math.log(z);
- arr.F.l1normalize(p);
-
- double count = edge.getCount();
- //increment expected count
- TIntArrayList phrase= edge.getPhrase();
- for(int tag=0;tag<K;tag++){
-
- exp_emit[tag][0][phrase.get(0)]+=p[tag]*count;
- exp_emit[tag][1][phrase.get(phrase.size()-1)]+=p[tag]*count;
-
- exp_pi[context][tag]+=p[tag]*count;
- }
- }
- }
-
- //System.out.println("Log likelihood: "+loglikelihood);
-
- //M
- for(double [][]i:exp_emit){
- for(double []j:i){
- arr.F.l1normalize(j);
- }
- }
-
- emit=exp_emit;
-
- for(double []j:exp_pi){
- arr.F.l1normalize(j);
- }
-
- pi=exp_pi;
-
- return loglikelihood;
- }
-
- public double[] posterior(Corpus.Edge edge)
- {
- double[] prob=Arrays.copyOf(pi[edge.getContextId()], K);
-
- TIntArrayList phrase = edge.getPhrase();
- TIntArrayList offsets = c.phraseEdges(phrase);
- for(int tag=0;tag<K;tag++)
- {
- for (int i=0; i < offsets.size(); ++i)
- prob[tag]*=emit[tag][i][phrase.get(offsets.get(i))];
- }
-
- return prob;
- }
-
- public void displayPosterior(PrintStream ps)
- {
- for (Edge edge : c.getEdges())
- {
- double probs[] = posterior(edge);
- arr.F.l1normalize(probs);
-
- // emit phrase
- ps.print(edge.getPhraseString());
- ps.print("\t");
- ps.print(edge.getContextString(true));
- int t=arr.F.argmax(probs);
- ps.println(" ||| C=" + t);
- }
- }
-
- public void displayModelParam(PrintStream ps)
- {
- final double EPS = 1e-6;
-
- ps.println("P(tag|context)");
- for (int i = 0; i < n_contexts; ++i)
- {
- ps.print(c.getContext(i));
- for(int j=0;j<pi[i].length;j++){
- if (pi[i][j] > EPS)
- ps.print("\t" + j + ": " + pi[i][j]);
- }
- ps.println();
- }
-
- ps.println("P(word|tag,position)");
- for (int i = 0; i < K; ++i)
- {
- for(int position=0;position<n_positions;position++){
- ps.println("tag " + i + " position " + position);
- for(int word=0;word<emit[i][position].length;word++){
- if (emit[i][position][word] > EPS)
- ps.print(c.getWord(word)+"="+emit[i][position][word]+"\t");
- }
- ps.println();
- }
- ps.println();
- }
-
- }
-
-}
diff --git a/gi/posterior-regularisation/prjava/src/phrase/Corpus.java b/gi/posterior-regularisation/prjava/src/phrase/Corpus.java deleted file mode 100644 index 4b1939cd..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/Corpus.java +++ /dev/null @@ -1,288 +0,0 @@ -package phrase; - -import gnu.trove.TIntArrayList; - -import java.io.*; -import java.util.*; -import java.util.regex.Pattern; - - -public class Corpus -{ - private Lexicon<String> wordLexicon = new Lexicon<String>(); - private Lexicon<TIntArrayList> phraseLexicon = new Lexicon<TIntArrayList>(); - private Lexicon<TIntArrayList> contextLexicon = new Lexicon<TIntArrayList>(); - private List<Edge> edges = new ArrayList<Edge>(); - private List<List<Edge>> phraseToContext = new ArrayList<List<Edge>>(); - private List<List<Edge>> contextToPhrase = new ArrayList<List<Edge>>(); - public int splitSentinel; - public int phraseSentinel; - public int rareSentinel; - - public Corpus() - { - splitSentinel = wordLexicon.insert("<SPLIT>"); - phraseSentinel = wordLexicon.insert("<PHRASE>"); - rareSentinel = wordLexicon.insert("<RARE>"); - } - - public class Edge - { - - Edge(int phraseId, int contextId, double count,int tag) - { - this.phraseId = phraseId; - this.contextId = contextId; - this.count = count; - fixTag=tag; - } - - Edge(int phraseId, int contextId, double count) - { - this.phraseId = phraseId; - this.contextId = contextId; - this.count = count; - fixTag=-1; - } - public int getTag(){ - return fixTag; - } - - public int getPhraseId() - { - return phraseId; - } - public TIntArrayList getPhrase() - { - return Corpus.this.getPhrase(phraseId); - } - public String getPhraseString() - { - return Corpus.this.getPhraseString(phraseId); - } - public int getContextId() - { - return contextId; - } - public TIntArrayList getContext() - { - return Corpus.this.getContext(contextId); - } - public String getContextString(boolean insertPhraseSentinel) - { - return Corpus.this.getContextString(contextId, insertPhraseSentinel); - } - public double getCount() - { - return count; - } - public boolean equals(Object other) - { - if (other instanceof Edge) - { - Edge oe = (Edge) other; - return oe.phraseId == phraseId && oe.contextId == contextId; - } - else return false; - } - public int hashCode() - { // this is how boost's hash_combine does it - int seed = phraseId; - seed ^= contextId + 0x9e3779b9 + (seed << 6) + (seed >> 2); - return seed; - } - public String toString() - { - return getPhraseString() + "\t" + getContextString(true); - } - - private int phraseId; - private int contextId; - private double count; - private int fixTag; - } - - List<Edge> getEdges() - { - return edges; - } - - int getNumEdges() - { - return edges.size(); - } - - int getNumPhrases() - { - return phraseLexicon.size(); - } - - int getNumContextPositions() - { - return contextLexicon.lookup(0).size(); - } - - List<Edge> getEdgesForPhrase(int phraseId) - { - return phraseToContext.get(phraseId); - } - - int getNumContexts() - { - return contextLexicon.size(); - } - - List<Edge> getEdgesForContext(int contextId) - { - return contextToPhrase.get(contextId); - } - - int getNumWords() - { - return wordLexicon.size(); - } - - String getWord(int wordId) - { - return wordLexicon.lookup(wordId); - } - - public TIntArrayList getPhrase(int phraseId) - { - return phraseLexicon.lookup(phraseId); - } - - public String getPhraseString(int phraseId) - { - StringBuffer b = new StringBuffer(); - for (int tid: getPhrase(phraseId).toNativeArray()) - { - if (b.length() > 0) - b.append(" "); - b.append(wordLexicon.lookup(tid)); - } - return b.toString(); - } - - public TIntArrayList getContext(int contextId) - { - return contextLexicon.lookup(contextId); - } - - public String getContextString(int contextId, boolean insertPhraseSentinel) - { - StringBuffer b = new StringBuffer(); - TIntArrayList c = getContext(contextId); - for (int i = 0; i < c.size(); ++i) - { - if (i > 0) b.append(" "); - //if (i == c.size() / 2) b.append("<PHRASE> "); - b.append(wordLexicon.lookup(c.get(i))); - } - return b.toString(); - } - - public boolean isSentinel(int wordId) - { - return wordId == splitSentinel || wordId == phraseSentinel; - } - - List<Edge> readEdges(Reader in) throws IOException - { - // read in line-by-line - BufferedReader bin = new BufferedReader(in); - String line; - Pattern separator = Pattern.compile(" \\|\\|\\| "); - - List<Edge> edges = new ArrayList<Edge>(); - while ((line = bin.readLine()) != null) - { - // split into phrase and contexts - StringTokenizer st = new StringTokenizer(line, "\t"); - assert (st.hasMoreTokens()); - String phraseToks = st.nextToken(); - assert (st.hasMoreTokens()); - String rest = st.nextToken(); - assert (!st.hasMoreTokens()); - - // process phrase - st = new StringTokenizer(phraseToks, " "); - TIntArrayList ptoks = new TIntArrayList(); - while (st.hasMoreTokens()) - ptoks.add(wordLexicon.insert(st.nextToken())); - int phraseId = phraseLexicon.insert(ptoks); - - // process contexts - String[] parts = separator.split(rest); - assert (parts.length % 2 == 0); - for (int i = 0; i < parts.length; i += 2) - { - // process pairs of strings - context and count - String ctxString = parts[i]; - String countString = parts[i + 1]; - - assert (countString.startsWith("C=")); - - String []countToks=countString.split(" "); - - double count = Double.parseDouble(countToks[0].substring(2).trim()); - - TIntArrayList ctx = new TIntArrayList(); - StringTokenizer ctxStrtok = new StringTokenizer(ctxString, " "); - while (ctxStrtok.hasMoreTokens()) - { - String token = ctxStrtok.nextToken(); - ctx.add(wordLexicon.insert(token)); - } - int contextId = contextLexicon.insert(ctx); - - - if(countToks.length<2){ - edges.add(new Edge(phraseId, contextId, count)); - } - else{ - int tag=Integer.parseInt(countToks[1].substring(2)); - edges.add(new Edge(phraseId, contextId, count,tag)); - } - } - } - return edges; - } - - static Corpus readFromFile(Reader in) throws IOException - { - Corpus c = new Corpus(); - c.edges = c.readEdges(in); - for (Edge edge: c.edges) - { - while (edge.getPhraseId() >= c.phraseToContext.size()) - c.phraseToContext.add(new ArrayList<Edge>()); - while (edge.getContextId() >= c.contextToPhrase.size()) - c.contextToPhrase.add(new ArrayList<Edge>()); - - // index the edge for fast phrase, context lookup - c.phraseToContext.get(edge.getPhraseId()).add(edge); - c.contextToPhrase.get(edge.getContextId()).add(edge); - } - return c; - } - - TIntArrayList phraseEdges(TIntArrayList phrase) - { - TIntArrayList r = new TIntArrayList(4); - for (int p = 0; p < phrase.size(); ++p) - { - if (p == 0 || phrase.get(p-1) == splitSentinel) - r.add(p); - if (p == phrase.size() - 1 || phrase.get(p+1) == splitSentinel) - r.add(p); - } - return r; - } - - public void printStats(PrintStream out) - { - out.println("Corpus has " + edges.size() + " edges " + phraseLexicon.size() + " phrases " - + contextLexicon.size() + " contexts and " + wordLexicon.size() + " word types"); - } -}
\ No newline at end of file diff --git a/gi/posterior-regularisation/prjava/src/phrase/Lexicon.java b/gi/posterior-regularisation/prjava/src/phrase/Lexicon.java deleted file mode 100644 index a386e4a3..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/Lexicon.java +++ /dev/null @@ -1,34 +0,0 @@ -package phrase; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class Lexicon<T> -{ - public int insert(T word) - { - Integer i = wordToIndex.get(word); - if (i == null) - { - i = indexToWord.size(); - wordToIndex.put(word, i); - indexToWord.add(word); - } - return i; - } - - public T lookup(int index) - { - return indexToWord.get(index); - } - - public int size() - { - return indexToWord.size(); - } - - private Map<T, Integer> wordToIndex = new HashMap<T, Integer>(); - private List<T> indexToWord = new ArrayList<T>(); -}
\ No newline at end of file diff --git a/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java b/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java deleted file mode 100644 index c032bb2b..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/PhraseCluster.java +++ /dev/null @@ -1,540 +0,0 @@ -package phrase;
-
-import gnu.trove.TIntArrayList;
-import org.apache.commons.math.special.Gamma;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.regex.Pattern;
-
-import phrase.Corpus.Edge;
-
-
-public class PhraseCluster {
-
- public int K;
- private int n_phrases, n_words, n_contexts, n_positions;
- public Corpus c;
- public ExecutorService pool;
-
- double[] lambdaPTCT;
- double[][] lambdaPT;
- boolean cacheLambda = true;
-
- // emit[tag][position][word] = p(word | tag, position in context)
- double emit[][][];
- // pi[phrase][tag] = p(tag | phrase)
- double pi[][];
-
- public PhraseCluster(int numCluster, Corpus corpus)
- {
- K=numCluster;
- c=corpus;
- n_words=c.getNumWords();
- n_phrases=c.getNumPhrases();
- n_contexts=c.getNumContexts();
- n_positions=c.getNumContextPositions();
-
- emit=new double [K][n_positions][n_words];
- pi=new double[n_phrases][K];
-
- for(double [][]i:emit)
- for(double []j:i)
- arr.F.randomise(j, true);
-
- for(double []j:pi)
- arr.F.randomise(j, true);
- }
-
- void useThreadPool(ExecutorService pool)
- {
- this.pool = pool;
- }
-
- public double EM(int phraseSizeLimit)
- {
- double [][][]exp_emit=new double [K][n_positions][n_words];
- double []exp_pi=new double[K];
-
- for(double [][]i:exp_emit)
- for(double []j:i)
- Arrays.fill(j, 1e-10);
-
- double loglikelihood=0;
-
- //E
- for(int phrase=0; phrase < n_phrases; phrase++)
- {
- if (phraseSizeLimit >= 1 && c.getPhrase(phrase).size() > phraseSizeLimit)
- continue;
-
- Arrays.fill(exp_pi, 1e-10);
-
- List<Edge> contexts = c.getEdgesForPhrase(phrase);
-
- for (int ctx=0; ctx<contexts.size(); ctx++)
- {
- Edge edge = contexts.get(ctx);
-
- double p[]=posterior(edge);
- double z = arr.F.l1norm(p);
- assert z > 0;
- loglikelihood += edge.getCount() * Math.log(z);
- arr.F.l1normalize(p);
-
- double count = edge.getCount();
- //increment expected count
- TIntArrayList context = edge.getContext();
- for(int tag=0;tag<K;tag++)
- {
- for(int pos=0;pos<n_positions;pos++){
- exp_emit[tag][pos][context.get(pos)]+=p[tag]*count;
- }
- exp_pi[tag]+=p[tag]*count;
- }
- }
- arr.F.l1normalize(exp_pi);
- System.arraycopy(exp_pi, 0, pi[phrase], 0, K);
- }
-
- //M
- for(double [][]i:exp_emit)
- for(double []j:i)
- arr.F.l1normalize(j);
-
- emit=exp_emit;
-
- return loglikelihood;
- }
-
- public double PREM(double scalePT, double scaleCT, int phraseSizeLimit)
- {
- if (scaleCT == 0)
- {
- if (pool != null)
- return PREM_phrase_constraints_parallel(scalePT, phraseSizeLimit);
- else
- return PREM_phrase_constraints(scalePT, phraseSizeLimit);
- }
- else // FIXME: ignores phraseSizeLimit
- return this.PREM_phrase_context_constraints(scalePT, scaleCT);
- }
-
-
- public double PREM_phrase_constraints(double scalePT, int phraseSizeLimit)
- {
- double [][][]exp_emit=new double[K][n_positions][n_words];
- double []exp_pi=new double[K];
-
- for(double [][]i:exp_emit)
- for(double []j:i)
- Arrays.fill(j, 1e-10);
-
- if (lambdaPT == null && cacheLambda)
- lambdaPT = new double[n_phrases][];
-
- double loglikelihood=0, kl=0, l1lmax=0, primal=0;
- int failures=0, iterations=0;
- long start = System.currentTimeMillis();
- //E
- for(int phrase=0; phrase<n_phrases; phrase++)
- {
- if (phraseSizeLimit >= 1 && c.getPhrase(phrase).size() > phraseSizeLimit)
- {
- //System.arraycopy(pi[phrase], 0, exp_pi[phrase], 0, K);
- continue;
- }
-
- Arrays.fill(exp_pi, 1e-10);
-
- // FIXME: add rare edge check to phrase objective & posterior processing
- PhraseObjective po = new PhraseObjective(this, phrase, scalePT, (cacheLambda) ? lambdaPT[phrase] : null);
- boolean ok = po.optimizeWithProjectedGradientDescent();
- if (!ok) ++failures;
- if (cacheLambda) lambdaPT[phrase] = po.getParameters();
- iterations += po.getNumberUpdateCalls();
- double [][] q=po.posterior();
- loglikelihood += po.loglikelihood();
- kl += po.KL_divergence();
- l1lmax += po.l1lmax();
- primal += po.primal(scalePT);
- List<Edge> edges = c.getEdgesForPhrase(phrase);
-
- for(int edge=0;edge<q.length;edge++){
- Edge e = edges.get(edge);
- TIntArrayList context = e.getContext();
- double contextCnt = e.getCount();
- //increment expected count
- for(int tag=0;tag<K;tag++){
- for(int pos=0;pos<n_positions;pos++){
- exp_emit[tag][pos][context.get(pos)]+=q[edge][tag]*contextCnt;
- }
-
- exp_pi[tag]+=q[edge][tag]*contextCnt;
-
- }
- }
- arr.F.l1normalize(exp_pi);
- System.arraycopy(exp_pi, 0, pi[phrase], 0, K);
- }
-
- long end = System.currentTimeMillis();
- if (failures > 0)
- System.out.println("WARNING: failed to converge in " + failures + "/" + n_phrases + " cases");
- System.out.println("\tmean iters: " + iterations/(double)n_phrases + " elapsed time " + (end - start) / 1000.0);
- System.out.println("\tllh: " + loglikelihood);
- System.out.println("\tKL: " + kl);
- System.out.println("\tphrase l1lmax: " + l1lmax);
-
- //M
- for(double [][]i:exp_emit)
- for(double []j:i)
- arr.F.l1normalize(j);
- emit=exp_emit;
-
- return primal;
- }
-
- public double PREM_phrase_constraints_parallel(final double scalePT, int phraseSizeLimit)
- {
- assert(pool != null);
-
- final LinkedBlockingQueue<PhraseObjective> expectations
- = new LinkedBlockingQueue<PhraseObjective>();
-
- double [][][]exp_emit=new double [K][n_positions][n_words];
- double [][]exp_pi=new double[n_phrases][K];
-
- for(double [][]i:exp_emit)
- for(double []j:i)
- Arrays.fill(j, 1e-10);
- for(double []j:exp_pi)
- Arrays.fill(j, 1e-10);
-
- double loglikelihood=0, kl=0, l1lmax=0, primal=0;
- final AtomicInteger failures = new AtomicInteger(0);
- final AtomicLong elapsed = new AtomicLong(0l);
- int iterations=0;
- long start = System.currentTimeMillis();
- List<Future<PhraseObjective>> results = new ArrayList<Future<PhraseObjective>>();
-
- if (lambdaPT == null && cacheLambda)
- lambdaPT = new double[n_phrases][];
-
- //E
- for(int phrase=0;phrase<n_phrases;phrase++) {
- if (phraseSizeLimit >= 1 && c.getPhrase(phrase).size() > phraseSizeLimit) {
- System.arraycopy(pi[phrase], 0, exp_pi[phrase], 0, K);
- continue;
- }
-
- final int p=phrase;
- results.add(pool.submit(new Callable<PhraseObjective>() {
- public PhraseObjective call() {
- //System.out.println("" + Thread.currentThread().getId() + " optimising lambda for " + p);
- long start = System.currentTimeMillis();
- PhraseObjective po = new PhraseObjective(PhraseCluster.this, p, scalePT, (cacheLambda) ? lambdaPT[p] : null);
- boolean ok = po.optimizeWithProjectedGradientDescent();
- if (!ok) failures.incrementAndGet();
- long end = System.currentTimeMillis();
- elapsed.addAndGet(end - start);
- //System.out.println("" + Thread.currentThread().getId() + " done optimising lambda for " + p);
- return po;
- }
- }));
- }
-
- // aggregate the expectations as they become available
- for (Future<PhraseObjective> fpo : results)
- {
- try {
- //System.out.println("" + Thread.currentThread().getId() + " reading queue #" + count);
-
- // wait (blocking) until something is ready
- PhraseObjective po = fpo.get();
- // process
- int phrase = po.phrase;
- if (cacheLambda) lambdaPT[phrase] = po.getParameters();
- //System.out.println("" + Thread.currentThread().getId() + " taken phrase " + phrase);
- double [][] q=po.posterior();
- loglikelihood += po.loglikelihood();
- kl += po.KL_divergence();
- l1lmax += po.l1lmax();
- primal += po.primal(scalePT);
- iterations += po.getNumberUpdateCalls();
-
- List<Edge> edges = c.getEdgesForPhrase(phrase);
- for(int edge=0;edge<q.length;edge++){
- Edge e = edges.get(edge);
- TIntArrayList context = e.getContext();
- double contextCnt = e.getCount();
- //increment expected count
- for(int tag=0;tag<K;tag++){
- for(int pos=0;pos<n_positions;pos++){
- exp_emit[tag][pos][context.get(pos)]+=q[edge][tag]*contextCnt;
- }
- exp_pi[phrase][tag]+=q[edge][tag]*contextCnt;
- }
- }
- } catch (InterruptedException e) {
- System.err.println("M-step thread interrupted. Probably fatal!");
- throw new RuntimeException(e);
- } catch (ExecutionException e) {
- System.err.println("M-step thread execution died. Probably fatal!");
- throw new RuntimeException(e);
- }
- }
-
- long end = System.currentTimeMillis();
-
- if (failures.get() > 0)
- System.out.println("WARNING: failed to converge in " + failures.get() + "/" + n_phrases + " cases");
- System.out.println("\tmean iters: " + iterations/(double)n_phrases + " walltime " + (end-start)/1000.0 + " threads " + elapsed.get() / 1000.0);
- System.out.println("\tllh: " + loglikelihood);
- System.out.println("\tKL: " + kl);
- System.out.println("\tphrase l1lmax: " + l1lmax);
-
- //M
- for(double [][]i:exp_emit)
- for(double []j:i)
- arr.F.l1normalize(j);
- emit=exp_emit;
-
- for(double []j:exp_pi)
- arr.F.l1normalize(j);
- pi=exp_pi;
-
- return primal;
- }
-
- public double PREM_phrase_context_constraints(double scalePT, double scaleCT)
- {
- double[][][] exp_emit = new double [K][n_positions][n_words];
- double[][] exp_pi = new double[n_phrases][K];
-
- //E step
- PhraseContextObjective pco = new PhraseContextObjective(this, lambdaPTCT, pool, scalePT, scaleCT);
- boolean ok = pco.optimizeWithProjectedGradientDescent();
- if (cacheLambda) lambdaPTCT = pco.getParameters();
-
- //now extract expectations
- List<Corpus.Edge> edges = c.getEdges();
- for(int e = 0; e < edges.size(); ++e)
- {
- double [] q = pco.posterior(e);
- Corpus.Edge edge = edges.get(e);
-
- TIntArrayList context = edge.getContext();
- double contextCnt = edge.getCount();
- //increment expected count
- for(int tag=0;tag<K;tag++)
- {
- for(int pos=0;pos<n_positions;pos++)
- exp_emit[tag][pos][context.get(pos)]+=q[tag]*contextCnt;
- exp_pi[edge.getPhraseId()][tag]+=q[tag]*contextCnt;
- }
- }
-
- System.out.println("\tllh: " + pco.loglikelihood());
- System.out.println("\tKL: " + pco.KL_divergence());
- System.out.println("\tphrase l1lmax: " + pco.phrase_l1lmax());
- System.out.println("\tcontext l1lmax: " + pco.context_l1lmax());
-
- //M step
- for(double [][]i:exp_emit)
- for(double []j:i)
- arr.F.l1normalize(j);
- emit=exp_emit;
-
- for(double []j:exp_pi)
- arr.F.l1normalize(j);
- pi=exp_pi;
-
- return pco.primal();
- }
-
- /**
- * @param phrase index of phrase
- * @param ctx array of context
- * @return unnormalized posterior
- */
- public double[] posterior(Corpus.Edge edge)
- {
- double[] prob;
-
- if(edge.getTag()>=0){
- prob=new double[K];
- prob[edge.getTag()]=1;
- return prob;
- }
-
- if (edge.getPhraseId() < n_phrases)
- prob = Arrays.copyOf(pi[edge.getPhraseId()], K);
- else
- {
- prob = new double[K];
- Arrays.fill(prob, 1.0);
- }
-
- TIntArrayList ctx = edge.getContext();
- for(int tag=0;tag<K;tag++)
- {
- for(int c=0;c<n_positions;c++)
- {
- int word = ctx.get(c);
- if (!this.c.isSentinel(word) && word < n_words)
- prob[tag]*=emit[tag][c][word];
- }
- }
-
- return prob;
- }
-
- public void displayPosterior(PrintStream ps, List<Edge> testing)
- {
- for (Edge edge : testing)
- {
- double probs[] = posterior(edge);
- arr.F.l1normalize(probs);
-
- // emit phrase
- ps.print(edge.getPhraseString());
- ps.print("\t");
- ps.print(edge.getContextString(true));
- int t=arr.F.argmax(probs);
- ps.println(" ||| C=" + t + " T=" + edge.getCount() + " P=" + probs[t]);
- //ps.println("# probs " + Arrays.toString(probs));
- }
- }
-
- public void displayModelParam(PrintStream ps)
- {
- final double EPS = 1e-6;
- ps.println("phrases " + n_phrases + " tags " + K + " positions " + n_positions);
-
- for (int i = 0; i < n_phrases; ++i)
- for(int j=0;j<pi[i].length;j++)
- if (pi[i][j] > EPS)
- ps.println(i + " " + j + " " + pi[i][j]);
-
- ps.println();
- for (int i = 0; i < K; ++i)
- {
- for(int position=0;position<n_positions;position++)
- {
- for(int word=0;word<emit[i][position].length;word++)
- {
- if (emit[i][position][word] > EPS)
- ps.println(i + " " + position + " " + word + " " + emit[i][position][word]);
- }
- }
- }
- }
-
- double phrase_l1lmax()
- {
- double sum=0;
- for(int phrase=0; phrase<n_phrases; phrase++)
- {
- double [] maxes = new double[K];
- for (Edge edge : c.getEdgesForPhrase(phrase))
- {
- double p[] = posterior(edge);
- arr.F.l1normalize(p);
- for(int tag=0;tag<K;tag++)
- maxes[tag] = Math.max(maxes[tag], p[tag]);
- }
- for(int tag=0;tag<K;tag++)
- sum += maxes[tag];
- }
- return sum;
- }
-
- double context_l1lmax()
- {
- double sum=0;
- for(int context=0; context<n_contexts; context++)
- {
- double [] maxes = new double[K];
- for (Edge edge : c.getEdgesForContext(context))
- {
- double p[] = posterior(edge);
- arr.F.l1normalize(p);
- for(int tag=0;tag<K;tag++)
- maxes[tag] = Math.max(maxes[tag], p[tag]);
- }
- for(int tag=0;tag<K;tag++)
- sum += maxes[tag];
- }
- return sum;
- }
-
- public void loadParameters(BufferedReader input) throws IOException
- {
- final double EPS = 1e-50;
-
- // overwrite pi, emit with ~zeros
- for(double [][]i:emit)
- for(double []j:i)
- Arrays.fill(j, EPS);
-
- for(double []j:pi)
- Arrays.fill(j, EPS);
-
- String line = input.readLine();
- assert line != null;
-
- Pattern space = Pattern.compile(" +");
- String[] parts = space.split(line);
- assert parts.length == 6;
-
- assert parts[0].equals("phrases");
- int phrases = Integer.parseInt(parts[1]);
- int tags = Integer.parseInt(parts[3]);
- int positions = Integer.parseInt(parts[5]);
-
- assert phrases == n_phrases;
- assert tags == K;
- assert positions == n_positions;
-
- // read in pi
- while ((line = input.readLine()) != null)
- {
- line = line.trim();
- if (line.isEmpty()) break;
-
- String[] tokens = space.split(line);
- assert tokens.length == 3;
- int p = Integer.parseInt(tokens[0]);
- int t = Integer.parseInt(tokens[1]);
- double v = Double.parseDouble(tokens[2]);
-
- pi[p][t] = v;
- }
-
- // read in emissions
- while ((line = input.readLine()) != null)
- {
- String[] tokens = space.split(line);
- assert tokens.length == 4;
- int t = Integer.parseInt(tokens[0]);
- int p = Integer.parseInt(tokens[1]);
- int w = Integer.parseInt(tokens[2]);
- double v = Double.parseDouble(tokens[3]);
-
- emit[t][p][w] = v;
- }
- }
-}
diff --git a/gi/posterior-regularisation/prjava/src/phrase/PhraseContextObjective.java b/gi/posterior-regularisation/prjava/src/phrase/PhraseContextObjective.java deleted file mode 100644 index 646ff392..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/PhraseContextObjective.java +++ /dev/null @@ -1,436 +0,0 @@ -package phrase;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-import optimization.gradientBasedMethods.ProjectedGradientDescent;
-import optimization.gradientBasedMethods.ProjectedObjective;
-import optimization.gradientBasedMethods.stats.OptimizerStats;
-import optimization.linesearch.ArmijoLineSearchMinimizationAlongProjectionArc;
-import optimization.linesearch.InterpolationPickFirstStep;
-import optimization.linesearch.LineSearchMethod;
-import optimization.projections.SimplexProjection;
-import optimization.stopCriteria.CompositeStopingCriteria;
-import optimization.stopCriteria.ProjectedGradientL2Norm;
-import optimization.stopCriteria.StopingCriteria;
-import optimization.stopCriteria.ValueDifference;
-import optimization.util.MathUtils;
-import phrase.Corpus.Edge;
-
-public class PhraseContextObjective extends ProjectedObjective
-{
- private static final double GRAD_DIFF = 0.00002;
- private static double INIT_STEP_SIZE = 300;
- private static double VAL_DIFF = 1e-8;
- private static int ITERATIONS = 20;
- boolean debug = false;
-
- private PhraseCluster c;
-
- // un-regularized unnormalized posterior, p[edge][tag]
- // P(tag|edge) \propto P(tag|phrase)P(context|tag)
- private double p[][];
-
- // regularized unnormalized posterior
- // q[edge][tag] propto p[edge][tag]*exp(-lambda)
- private double q[][];
- private List<Corpus.Edge> data;
-
- // log likelihood under q
- private double loglikelihood;
- private SimplexProjection projectionPhrase;
- private SimplexProjection projectionContext;
-
- double[] newPoint;
- private int n_param;
-
- // likelihood under p
- public double llh;
-
- private static Map<Corpus.Edge, Integer> edgeIndex;
-
- private long projectionTime;
- private long objectiveTime;
- private long actualProjectionTime;
- private ExecutorService pool;
-
- double scalePT;
- double scaleCT;
-
- public PhraseContextObjective(PhraseCluster cluster, double[] startingParameters, ExecutorService pool,
- double scalePT, double scaleCT)
- {
- c=cluster;
- data=c.c.getEdges();
- n_param=data.size()*c.K*2;
- this.pool=pool;
- this.scalePT = scalePT;
- this.scaleCT = scaleCT;
-
- parameters = startingParameters;
- if (parameters == null)
- parameters = new double[n_param];
-
- System.out.println("Num parameters " + n_param);
- newPoint = new double[n_param];
- gradient = new double[n_param];
- initP();
- projectionPhrase = new SimplexProjection(scalePT);
- projectionContext = new SimplexProjection(scaleCT);
- q=new double [data.size()][c.K];
-
- if (edgeIndex == null) {
- edgeIndex = new HashMap<Edge, Integer>();
- for (int e=0; e<data.size(); e++)
- {
- edgeIndex.put(data.get(e), e);
- //if (debug) System.out.println("Edge " + data.get(e) + " index " + e);
- }
- }
-
- setParameters(parameters);
- }
-
- private void initP(){
- p=new double[data.size()][];
- for(int edge=0;edge<data.size();edge++)
- {
- p[edge]=c.posterior(data.get(edge));
- llh += data.get(edge).getCount() * Math.log(arr.F.l1norm(p[edge]));
- arr.F.l1normalize(p[edge]);
- }
- }
-
- @Override
- public void setParameters(double[] params) {
- //System.out.println("setParameters " + Arrays.toString(parameters));
- // TODO: test if params have changed and skip update otherwise
- super.setParameters(params);
- updateFunction();
- }
-
- private void updateFunction()
- {
- updateCalls++;
- loglikelihood=0;
-
- System.out.print(".");
- System.out.flush();
-
- long begin = System.currentTimeMillis();
- for (int e=0; e<data.size(); e++)
- {
- Edge edge = data.get(e);
- for(int tag=0; tag<c.K; tag++)
- {
- int ip = index(e, tag, true);
- int ic = index(e, tag, false);
- q[e][tag] = p[e][tag]*
- Math.exp((-parameters[ip]-parameters[ic]) / edge.getCount());
- //if (debug)
- //System.out.println("\tposterior " + edge + " with tag " + tag + " p " + p[e][tag] + " params " + parameters[ip] + " and " + parameters[ic] + " q " + q[e][tag]);
- }
- }
-
- for(int edge=0;edge<data.size();edge++) {
- loglikelihood+=data.get(edge).getCount() * Math.log(arr.F.l1norm(q[edge]));
- arr.F.l1normalize(q[edge]);
- }
-
- for (int e=0; e<data.size(); e++)
- {
- for(int tag=0; tag<c.K; tag++)
- {
- int ip = index(e, tag, true);
- int ic = index(e, tag, false);
- gradient[ip]=-q[e][tag];
- gradient[ic]=-q[e][tag];
- }
- }
- //if (debug) {
- //System.out.println("objective " + loglikelihood + " ||gradient||_2: " + arr.F.l2norm(gradient));
- //System.out.println("gradient " + Arrays.toString(gradient));
- //}
- objectiveTime += System.currentTimeMillis() - begin;
- }
-
- @Override
- public double[] projectPoint(double[] point)
- {
- long begin = System.currentTimeMillis();
- List<Future<?>> tasks = new ArrayList<Future<?>>();
-
- System.out.print(",");
- System.out.flush();
-
- Arrays.fill(newPoint, 0, newPoint.length, 0);
-
- // first project using the phrase-tag constraints,
- // for all p,t: sum_c lambda_ptc < scaleP
- if (pool == null)
- {
- for (int p = 0; p < c.c.getNumPhrases(); ++p)
- {
- List<Edge> edges = c.c.getEdgesForPhrase(p);
- double[] toProject = new double[edges.size()];
- for(int tag=0;tag<c.K;tag++)
- {
- // FIXME: slow hash lookup for e (twice)
- for(int e=0; e<edges.size(); e++)
- toProject[e] = point[index(edges.get(e), tag, true)];
- long lbegin = System.currentTimeMillis();
- projectionPhrase.project(toProject);
- actualProjectionTime += System.currentTimeMillis() - lbegin;
- for(int e=0; e<edges.size(); e++)
- newPoint[index(edges.get(e), tag, true)] = toProject[e];
- }
- }
- }
- else // do above in parallel using thread pool
- {
- for (int p = 0; p < c.c.getNumPhrases(); ++p)
- {
- final int phrase = p;
- final double[] inPoint = point;
- Runnable task = new Runnable()
- {
- public void run()
- {
- List<Edge> edges = c.c.getEdgesForPhrase(phrase);
- double toProject[] = new double[edges.size()];
- for(int tag=0;tag<c.K;tag++)
- {
- // FIXME: slow hash lookup for e
- for(int e=0; e<edges.size(); e++)
- toProject[e] = inPoint[index(edges.get(e), tag, true)];
- projectionPhrase.project(toProject);
- for(int e=0; e<edges.size(); e++)
- newPoint[index(edges.get(e), tag, true)] = toProject[e];
- }
- }
- };
- tasks.add(pool.submit(task));
- }
- }
- //System.out.println("after PT " + Arrays.toString(newPoint));
-
- // now project using the context-tag constraints,
- // for all c,t: sum_p omega_pct < scaleC
- if (pool == null)
- {
- for (int ctx = 0; ctx < c.c.getNumContexts(); ++ctx)
- {
- List<Edge> edges = c.c.getEdgesForContext(ctx);
- double toProject[] = new double[edges.size()];
- for(int tag=0;tag<c.K;tag++)
- {
- // FIXME: slow hash lookup for e
- for(int e=0; e<edges.size(); e++)
- toProject[e] = point[index(edges.get(e), tag, false)];
- long lbegin = System.currentTimeMillis();
- projectionContext.project(toProject);
- actualProjectionTime += System.currentTimeMillis() - lbegin;
- for(int e=0; e<edges.size(); e++)
- newPoint[index(edges.get(e), tag, false)] = toProject[e];
- }
- }
- }
- else
- {
- // do above in parallel using thread pool
- for (int ctx = 0; ctx < c.c.getNumContexts(); ++ctx)
- {
- final int context = ctx;
- final double[] inPoint = point;
- Runnable task = new Runnable()
- {
- public void run()
- {
- List<Edge> edges = c.c.getEdgesForContext(context);
- double toProject[] = new double[edges.size()];
- for(int tag=0;tag<c.K;tag++)
- {
- // FIXME: slow hash lookup for e
- for(int e=0; e<edges.size(); e++)
- toProject[e] = inPoint[index(edges.get(e), tag, false)];
- projectionContext.project(toProject);
- for(int e=0; e<edges.size(); e++)
- newPoint[index(edges.get(e), tag, false)] = toProject[e];
- }
- }
- };
- tasks.add(pool.submit(task));
- }
- }
-
- if (pool != null)
- {
- // wait for all the jobs to complete
- Exception failure = null;
- for (Future<?> task: tasks)
- {
- try {
- task.get();
- } catch (InterruptedException e) {
- System.err.println("ERROR: Projection thread interrupted");
- e.printStackTrace();
- failure = e;
- } catch (ExecutionException e) {
- System.err.println("ERROR: Projection thread died");
- e.printStackTrace();
- failure = e;
- }
- }
- // rethrow the exception
- if (failure != null)
- {
- pool.shutdownNow();
- throw new RuntimeException(failure);
- }
- }
-
- double[] tmp = newPoint;
- newPoint = point;
- projectionTime += System.currentTimeMillis() - begin;
-
- //if (debug)
- //System.out.println("\t\treturning " + Arrays.toString(tmp));
- return tmp;
- }
-
- private int index(Edge edge, int tag, boolean phrase)
- {
- // NB if indexing changes must also change code in updateFunction and constructor
- if (phrase)
- return tag * edgeIndex.size() + edgeIndex.get(edge);
- else
- return (c.K + tag) * edgeIndex.size() + edgeIndex.get(edge);
- }
-
- private int index(int e, int tag, boolean phrase)
- {
- // NB if indexing changes must also change code in updateFunction and constructor
- if (phrase)
- return tag * edgeIndex.size() + e;
- else
- return (c.K + tag) * edgeIndex.size() + e;
- }
-
- @Override
- public double[] getGradient() {
- gradientCalls++;
- return gradient;
- }
-
- @Override
- public double getValue() {
- functionCalls++;
- return loglikelihood;
- }
-
- @Override
- public String toString() {
- return "No need for pointless toString";
- }
-
- public double []posterior(int edgeIndex){
- return q[edgeIndex];
- }
-
- public boolean optimizeWithProjectedGradientDescent()
- {
- projectionTime = 0;
- actualProjectionTime = 0;
- objectiveTime = 0;
- long start = System.currentTimeMillis();
-
- LineSearchMethod ls =
- new ArmijoLineSearchMinimizationAlongProjectionArc
- (new InterpolationPickFirstStep(INIT_STEP_SIZE));
- //LineSearchMethod ls = new WolfRuleLineSearch(
- // (new InterpolationPickFirstStep(INIT_STEP_SIZE)), c1, c2);
- OptimizerStats stats = new OptimizerStats();
-
-
- ProjectedGradientDescent optimizer = new ProjectedGradientDescent(ls);
- StopingCriteria stopGrad = new ProjectedGradientL2Norm(GRAD_DIFF);
- StopingCriteria stopValue = new ValueDifference(VAL_DIFF*(-llh));
- CompositeStopingCriteria compositeStop = new CompositeStopingCriteria();
- compositeStop.add(stopGrad);
- compositeStop.add(stopValue);
- optimizer.setMaxIterations(ITERATIONS);
- updateFunction();
- boolean success = optimizer.optimize(this,stats,compositeStop);
-
- System.out.println();
- System.out.println(stats.prettyPrint(1));
-
- if (success)
- System.out.print("\toptimization took " + optimizer.getCurrentIteration() + " iterations");
- else
- System.out.print("\toptimization failed to converge");
- long total = System.currentTimeMillis() - start;
- System.out.println(" and " + total + " ms: projection " + projectionTime +
- " actual " + actualProjectionTime + " objective " + objectiveTime);
-
- return success;
- }
-
- double loglikelihood()
- {
- return llh;
- }
-
- double KL_divergence()
- {
- return -loglikelihood + MathUtils.dotProduct(parameters, gradient);
- }
-
- double phrase_l1lmax()
- {
- // \sum_{tag,phrase} max_{context} P(tag|context,phrase)
- double sum=0;
- for (int p = 0; p < c.c.getNumPhrases(); ++p)
- {
- List<Edge> edges = c.c.getEdgesForPhrase(p);
- for(int tag=0;tag<c.K;tag++)
- {
- double max=0;
- for (Edge edge: edges)
- max = Math.max(max, q[edgeIndex.get(edge)][tag]);
- sum+=max;
- }
- }
- return sum;
- }
-
- double context_l1lmax()
- {
- // \sum_{tag,context} max_{phrase} P(tag|context,phrase)
- double sum=0;
- for (int ctx = 0; ctx < c.c.getNumContexts(); ++ctx)
- {
- List<Edge> edges = c.c.getEdgesForContext(ctx);
- for(int tag=0; tag<c.K; tag++)
- {
- double max=0;
- for (Edge edge: edges)
- max = Math.max(max, q[edgeIndex.get(edge)][tag]);
- sum+=max;
- }
- }
- return sum;
- }
-
- // L - KL(q||p) - scalePT * l1lmax_phrase - scaleCT * l1lmax_context
- public double primal()
- {
- return loglikelihood() - KL_divergence() - scalePT * phrase_l1lmax() - scaleCT * context_l1lmax();
- }
-}
\ No newline at end of file diff --git a/gi/posterior-regularisation/prjava/src/phrase/PhraseCorpus.java b/gi/posterior-regularisation/prjava/src/phrase/PhraseCorpus.java deleted file mode 100644 index 0cf31c1c..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/PhraseCorpus.java +++ /dev/null @@ -1,193 +0,0 @@ -package phrase;
-
-import io.FileUtil;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Scanner;
-
-public class PhraseCorpus
-{
- public HashMap<String,Integer>wordLex;
- public HashMap<String,Integer>phraseLex;
-
- public String wordList[];
- public String phraseList[];
-
- //data[phrase][num context][position]
- public int data[][][];
- public int numContexts;
-
- public PhraseCorpus(String filename) throws FileNotFoundException, IOException
- {
- BufferedReader r = FileUtil.reader(new File(filename));
-
- phraseLex=new HashMap<String,Integer>();
- wordLex=new HashMap<String,Integer>();
-
- ArrayList<int[][]>dataList=new ArrayList<int[][]>();
- String line=null;
- numContexts = 0;
-
- while((line=readLine(r))!=null){
-
- String toks[]=line.split("\t");
- String phrase=toks[0];
- addLex(phrase,phraseLex);
-
- toks=toks[1].split(" \\|\\|\\| ");
-
- ArrayList <int[]>ctxList=new ArrayList<int[]>();
-
- for(int i=0;i<toks.length;i+=2){
- String ctx=toks[i];
- String words[]=ctx.split(" ");
- if (numContexts == 0)
- numContexts = words.length - 1;
- else
- assert numContexts == words.length - 1;
-
- int []context=new int [numContexts+1];
- int idx=0;
- for(String word:words){
- if(word.equals("<PHRASE>")){
- continue;
- }
- addLex(word,wordLex);
- context[idx]=wordLex.get(word);
- idx++;
- }
-
- String count=toks[i+1];
- context[idx]=Integer.parseInt(count.trim().substring(2));
-
- ctxList.add(context);
- }
-
- dataList.add(ctxList.toArray(new int [0][]));
-
- }
- try{
- r.close();
- }catch(IOException ioe){
- ioe.printStackTrace();
- }
- data=dataList.toArray(new int[0][][]);
- }
-
- private void addLex(String key, HashMap<String,Integer>lex){
- Integer i=lex.get(key);
- if(i==null){
- lex.put(key, lex.size());
- }
- }
-
- //for debugging
- public void saveLex(String lexFilename) throws FileNotFoundException, IOException
- {
- PrintStream ps = FileUtil.printstream(new File(lexFilename));
- ps.println("Phrase Lexicon");
- ps.println(phraseLex.size());
- printDict(phraseLex,ps);
-
- ps.println("Word Lexicon");
- ps.println(wordLex.size());
- printDict(wordLex,ps);
- ps.close();
- }
-
- private static void printDict(HashMap<String,Integer>lex,PrintStream ps){
- String []dict=buildList(lex);
- for(int i=0;i<dict.length;i++){
- ps.println(dict[i]);
- }
- }
-
- public void loadLex(String lexFilename){
- Scanner sc=io.FileUtil.openInFile(lexFilename);
-
- sc.nextLine();
- int size=sc.nextInt();
- sc.nextLine();
- String[]dict=new String[size];
- for(int i=0;i<size;i++){
- dict[i]=sc.nextLine();
- }
- phraseLex=buildMap(dict);
-
- sc.nextLine();
- size=sc.nextInt();
- sc.nextLine();
- dict=new String[size];
- for(int i=0;i<size;i++){
- dict[i]=sc.nextLine();
- }
- wordLex=buildMap(dict);
- sc.close();
- }
-
- private HashMap<String, Integer> buildMap(String[]dict){
- HashMap<String,Integer> map=new HashMap<String,Integer>();
- for(int i=0;i<dict.length;i++){
- map.put(dict[i], i);
- }
- return map;
- }
-
- public void buildList(){
- if(wordList==null){
- wordList=buildList(wordLex);
- phraseList=buildList(phraseLex);
- }
- }
-
- private static String[]buildList(HashMap<String,Integer>lex){
- String dict[]=new String [lex.size()];
- for(String key:lex.keySet()){
- dict[lex.get(key)]=key;
- }
- return dict;
- }
-
- public String getContextString(int context[], boolean addPhraseMarker)
- {
- StringBuffer b = new StringBuffer();
- for (int i=0;i<context.length-1;i++)
- {
- if (b.length() > 0)
- b.append(" ");
-
- if (i == context.length/2)
- b.append("<PHRASE> ");
-
- b.append(wordList[context[i]]);
- }
- return b.toString();
- }
-
- public static String readLine(BufferedReader r){
- try{
- return r.readLine();
- }
- catch(IOException ioe){
- ioe.printStackTrace();
- }
- return null;
- }
-
- public static void main(String[] args) throws Exception
- {
- String LEX_FILENAME="../pdata/lex.out";
- String DATA_FILENAME="../pdata/btec.con";
- PhraseCorpus c=new PhraseCorpus(DATA_FILENAME);
- c.saveLex(LEX_FILENAME);
- c.loadLex(LEX_FILENAME);
- c.saveLex(LEX_FILENAME);
- }
-}
diff --git a/gi/posterior-regularisation/prjava/src/phrase/PhraseObjective.java b/gi/posterior-regularisation/prjava/src/phrase/PhraseObjective.java deleted file mode 100644 index ac73a075..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/PhraseObjective.java +++ /dev/null @@ -1,224 +0,0 @@ -package phrase;
-
-import java.util.Arrays;
-import java.util.List;
-
-import optimization.gradientBasedMethods.ProjectedGradientDescent;
-import optimization.gradientBasedMethods.ProjectedObjective;
-import optimization.gradientBasedMethods.stats.OptimizerStats;
-import optimization.linesearch.ArmijoLineSearchMinimizationAlongProjectionArc;
-import optimization.linesearch.InterpolationPickFirstStep;
-import optimization.linesearch.LineSearchMethod;
-import optimization.linesearch.WolfRuleLineSearch;
-import optimization.projections.SimplexProjection;
-import optimization.stopCriteria.CompositeStopingCriteria;
-import optimization.stopCriteria.ProjectedGradientL2Norm;
-import optimization.stopCriteria.StopingCriteria;
-import optimization.stopCriteria.ValueDifference;
-import optimization.util.MathUtils;
-
-public class PhraseObjective extends ProjectedObjective
-{
- static final double GRAD_DIFF = 0.00002;
- static double INIT_STEP_SIZE = 300;
- static double VAL_DIFF = 1e-8; // tuned to BTEC subsample
- static int ITERATIONS = 100;
- private PhraseCluster c;
-
- /**@brief
- * for debugging purposes
- */
- //public static PrintStream ps;
-
- /**@brief current phrase being optimzed*/
- public int phrase;
-
- /**@brief un-regularized posterior
- * unnormalized
- * p[edge][tag]
- * P(tag|edge) \propto P(tag|phrase)P(context|tag)
- */
- private double[][]p;
-
- /**@brief regularized posterior
- * q[edge][tag] propto p[edge][tag]*exp(-lambda)
- */
- private double q[][];
- private List<Corpus.Edge> data;
-
- /**@brief log likelihood of the associated phrase
- *
- */
- private double loglikelihood;
- private SimplexProjection projection;
-
- double[] newPoint ;
-
- private int n_param;
-
- /**@brief likelihood under p
- *
- */
- public double llh;
-
- public PhraseObjective(PhraseCluster cluster, int phraseIdx, double scale, double[] lambda){
- phrase=phraseIdx;
- c=cluster;
- data=c.c.getEdgesForPhrase(phrase);
- n_param=data.size()*c.K;
- //System.out.println("Num parameters " + n_param + " for phrase #" + phraseIdx);
-
- if (lambda==null)
- lambda=new double[n_param];
-
- parameters = lambda;
- newPoint = new double[n_param];
- gradient = new double[n_param];
- initP();
- projection=new SimplexProjection(scale);
- q=new double [data.size()][c.K];
-
- setParameters(parameters);
- }
-
- private void initP(){
- p=new double[data.size()][];
- for(int edge=0;edge<data.size();edge++){
- p[edge]=c.posterior(data.get(edge));
- llh += data.get(edge).getCount() * Math.log(arr.F.l1norm(p[edge])); // Was bug here - count inside log!
- arr.F.l1normalize(p[edge]);
- }
- }
-
- @Override
- public void setParameters(double[] params) {
- super.setParameters(params);
- updateFunction();
- }
-
- private void updateFunction(){
- updateCalls++;
- loglikelihood=0;
-
- for(int tag=0;tag<c.K;tag++){
- for(int edge=0;edge<data.size();edge++){
- q[edge][tag]=p[edge][tag]*
- Math.exp(-parameters[tag*data.size()+edge]/data.get(edge).getCount());
- }
- }
-
- for(int edge=0;edge<data.size();edge++){
- loglikelihood+=data.get(edge).getCount() * Math.log(arr.F.l1norm(q[edge]));
- arr.F.l1normalize(q[edge]);
- }
-
- for(int tag=0;tag<c.K;tag++){
- for(int edge=0;edge<data.size();edge++){
- gradient[tag*data.size()+edge]=-q[edge][tag];
- }
- }
- }
-
- @Override
- public double[] projectPoint(double[] point)
- {
- double toProject[]=new double[data.size()];
- for(int tag=0;tag<c.K;tag++){
- for(int edge=0;edge<data.size();edge++){
- toProject[edge]=point[tag*data.size()+edge];
- }
- projection.project(toProject);
- for(int edge=0;edge<data.size();edge++){
- newPoint[tag*data.size()+edge]=toProject[edge];
- }
- }
- return newPoint;
- }
-
- @Override
- public double[] getGradient() {
- gradientCalls++;
- return gradient;
- }
-
- @Override
- public double getValue() {
- functionCalls++;
- return loglikelihood;
- }
-
- @Override
- public String toString() {
- return Arrays.toString(parameters);
- }
-
- public double [][]posterior(){
- return q;
- }
-
- long optimizationTime;
-
- public boolean optimizeWithProjectedGradientDescent(){
- long start = System.currentTimeMillis();
-
- LineSearchMethod ls =
- new ArmijoLineSearchMinimizationAlongProjectionArc
- (new InterpolationPickFirstStep(INIT_STEP_SIZE));
- //LineSearchMethod ls = new WolfRuleLineSearch(
- // (new InterpolationPickFirstStep(INIT_STEP_SIZE)), c1, c2);
- OptimizerStats stats = new OptimizerStats();
-
-
- ProjectedGradientDescent optimizer = new ProjectedGradientDescent(ls);
- StopingCriteria stopGrad = new ProjectedGradientL2Norm(GRAD_DIFF);
- StopingCriteria stopValue = new ValueDifference(VAL_DIFF*(-llh));
- CompositeStopingCriteria compositeStop = new CompositeStopingCriteria();
- compositeStop.add(stopGrad);
- compositeStop.add(stopValue);
- optimizer.setMaxIterations(ITERATIONS);
- updateFunction();
- boolean success = optimizer.optimize(this,stats,compositeStop);
- //System.out.println("Ended optimzation Projected Gradient Descent\n" + stats.prettyPrint(1));
- //if(succed){
- //System.out.println("Ended optimization in " + optimizer.getCurrentIteration());
- //}else{
-// System.out.println("Failed to optimize");
- //}
- //System.out.println(Arrays.toString(parameters));
-
- // for(int edge=0;edge<data.getSize();edge++){
- // ps.println(Arrays.toString(q[edge]));
- // }
-
- return success;
- }
-
- public double KL_divergence()
- {
- return -loglikelihood + MathUtils.dotProduct(parameters, gradient);
- }
-
- public double loglikelihood()
- {
- return llh;
- }
-
- public double l1lmax()
- {
- double sum=0;
- for(int tag=0;tag<c.K;tag++){
- double max=0;
- for(int edge=0;edge<data.size();edge++){
- if(q[edge][tag]>max)
- max=q[edge][tag];
- }
- sum+=max;
- }
- return sum;
- }
-
- public double primal(double scale)
- {
- return loglikelihood() - KL_divergence() - scale * l1lmax();
- }
-}
diff --git a/gi/posterior-regularisation/prjava/src/phrase/Trainer.java b/gi/posterior-regularisation/prjava/src/phrase/Trainer.java deleted file mode 100644 index 6f302b20..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/Trainer.java +++ /dev/null @@ -1,257 +0,0 @@ -package phrase; - -import io.FileUtil; -import joptsimple.OptionParser; -import joptsimple.OptionSet; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import phrase.Corpus.Edge; - -import arr.F; - -public class Trainer -{ - public static void main(String[] args) - { - OptionParser parser = new OptionParser(); - parser.accepts("help"); - parser.accepts("in").withRequiredArg().ofType(File.class); - parser.accepts("in1").withRequiredArg().ofType(File.class); - parser.accepts("test").withRequiredArg().ofType(File.class); - parser.accepts("out").withRequiredArg().ofType(File.class); - parser.accepts("start").withRequiredArg().ofType(File.class); - parser.accepts("parameters").withRequiredArg().ofType(File.class); - parser.accepts("topics").withRequiredArg().ofType(Integer.class).defaultsTo(5); - parser.accepts("iterations").withRequiredArg().ofType(Integer.class).defaultsTo(10); - parser.accepts("threads").withRequiredArg().ofType(Integer.class).defaultsTo(0); - parser.accepts("scale-phrase").withRequiredArg().ofType(Double.class).defaultsTo(0.0); - parser.accepts("scale-context").withRequiredArg().ofType(Double.class).defaultsTo(0.0); - parser.accepts("seed").withRequiredArg().ofType(Long.class).defaultsTo(0l); - parser.accepts("convergence-threshold").withRequiredArg().ofType(Double.class).defaultsTo(1e-6); - parser.accepts("variational-bayes"); - parser.accepts("alpha-emit").withRequiredArg().ofType(Double.class).defaultsTo(0.1); - parser.accepts("alpha-pi").withRequiredArg().ofType(Double.class).defaultsTo(0.0001); - parser.accepts("agree-direction"); - parser.accepts("agree-language"); - parser.accepts("no-parameter-cache"); - parser.accepts("skip-large-phrases").withRequiredArg().ofType(Integer.class).defaultsTo(5); - OptionSet options = parser.parse(args); - - if (options.has("help") || !options.has("in")) - { - try { - parser.printHelpOn(System.err); - } catch (IOException e) { - System.err.println("This should never happen."); - e.printStackTrace(); - } - System.exit(1); - } - - int tags = (Integer) options.valueOf("topics"); - int iterations = (Integer) options.valueOf("iterations"); - double scale_phrase = (Double) options.valueOf("scale-phrase"); - double scale_context = (Double) options.valueOf("scale-context"); - int threads = (Integer) options.valueOf("threads"); - double threshold = (Double) options.valueOf("convergence-threshold"); - boolean vb = options.has("variational-bayes"); - double alphaEmit = (vb) ? (Double) options.valueOf("alpha-emit") : 0; - double alphaPi = (vb) ? (Double) options.valueOf("alpha-pi") : 0; - int skip = (Integer) options.valueOf("skip-large-phrases"); - - if (options.has("seed")) - F.rng = new Random((Long) options.valueOf("seed")); - - ExecutorService threadPool = null; - if (threads > 0) - threadPool = Executors.newFixedThreadPool(threads); - - if (tags <= 1 || scale_phrase < 0 || scale_context < 0 || threshold < 0) - { - System.err.println("Invalid arguments. Try again!"); - System.exit(1); - } - - Corpus corpus = null; - File infile = (File) options.valueOf("in"); - Corpus corpus1 = null; - File infile1 = (File) options.valueOf("in1"); - try { - System.out.println("Reading concordance from " + infile); - corpus = Corpus.readFromFile(FileUtil.reader(infile)); - corpus.printStats(System.out); - if(options.has("in1")){ - corpus1 = Corpus.readFromFile(FileUtil.reader(infile1)); - corpus1.printStats(System.out); - } - } catch (IOException e) { - System.err.println("Failed to open input file: " + infile); - e.printStackTrace(); - System.exit(1); - } - - if (!(options.has("agree-direction")||options.has("agree-language"))) - System.out.println("Running with " + tags + " tags " + - "for " + iterations + " iterations " + - ((skip > 0) ? "skipping large phrases for first " + skip + " iterations " : "") + - "with scale " + scale_phrase + " phrase and " + scale_context + " context " + - "and " + threads + " threads"); - else - System.out.println("Running agreement model with " + tags + " tags " + - "for " + iterations); - - System.out.println(); - - PhraseCluster cluster = null; - Agree2Sides agree2sides = null; - Agree agree= null; - VB vbModel=null; - if (options.has("agree-language")) - agree2sides = new Agree2Sides(tags, corpus,corpus1); - else if (options.has("agree-direction")) - agree = new Agree(tags, corpus); - else - { - if (vb) - { - vbModel=new VB(tags,corpus); - vbModel.alpha=alphaPi; - vbModel.lambda=alphaEmit; - if (threadPool != null) vbModel.useThreadPool(threadPool); - } - else - { - cluster = new PhraseCluster(tags, corpus); - if (threadPool != null) cluster.useThreadPool(threadPool); - - if (options.has("no-parameter-cache")) - cluster.cacheLambda = false; - if (options.has("start")) - { - try { - System.err.println("Reading starting parameters from " + options.valueOf("start")); - cluster.loadParameters(FileUtil.reader((File)options.valueOf("start"))); - } catch (IOException e) { - System.err.println("Failed to open input file: " + options.valueOf("start")); - e.printStackTrace(); - } - } - } - } - - double last = 0; - for (int i=0; i < iterations; i++) - { - double o; - if (agree != null) - o = agree.EM(); - else if(agree2sides!=null) - o = agree2sides.EM(); - else - { - if (i < skip) - System.out.println("Skipping phrases of length > " + (i+1)); - - if (scale_phrase <= 0 && scale_context <= 0) - { - if (!vb) - o = cluster.EM((i < skip) ? i+1 : 0); - else - o = vbModel.EM(); - } - else - o = cluster.PREM(scale_phrase, scale_context, (i < skip) ? i+1 : 0); - } - - System.out.println("ITER: "+i+" objective: " + o); - - // sometimes takes a few iterations to break the ties - if (i > 5 && Math.abs((o - last) / o) < threshold) - { - last = o; - break; - } - last = o; - } - - double pl1lmax = 0, cl1lmax = 0; - if (cluster != null) - { - pl1lmax = cluster.phrase_l1lmax(); - cl1lmax = cluster.context_l1lmax(); - } - else if (agree != null) - { - // fairly arbitrary choice of model1 cf model2 - pl1lmax = agree.model1.phrase_l1lmax(); - cl1lmax = agree.model1.context_l1lmax(); - } - else if (agree2sides != null) - { - // fairly arbitrary choice of model1 cf model2 - pl1lmax = agree2sides.model1.phrase_l1lmax(); - cl1lmax = agree2sides.model1.context_l1lmax(); - } - - System.out.println("\nFinal posterior phrase l1lmax " + pl1lmax + " context l1lmax " + cl1lmax); - - if (options.has("out")) - { - File outfile = (File) options.valueOf("out"); - try { - PrintStream ps = FileUtil.printstream(outfile); - List<Edge> test; - if (!options.has("test")) // just use the training - test = corpus.getEdges(); - else - { // if --test supplied, load up the file - infile = (File) options.valueOf("test"); - System.out.println("Reading testing concordance from " + infile); - test = corpus.readEdges(FileUtil.reader(infile)); - } - if(vb) { - assert !options.has("test"); - vbModel.displayPosterior(ps); - } else if (cluster != null) - cluster.displayPosterior(ps, test); - else if (agree != null) - agree.displayPosterior(ps, test); - else if (agree2sides != null) { - assert !options.has("test"); - agree2sides.displayPosterior(ps); - } - - ps.close(); - } catch (IOException e) { - System.err.println("Failed to open either testing file or output file"); - e.printStackTrace(); - System.exit(1); - } - } - - if (options.has("parameters")) - { - assert !vb; - File outfile = (File) options.valueOf("parameters"); - PrintStream ps; - try { - ps = FileUtil.printstream(outfile); - cluster.displayModelParam(ps); - ps.close(); - } catch (IOException e) { - System.err.println("Failed to open output parameters file: " + outfile); - e.printStackTrace(); - System.exit(1); - } - } - - if (cluster != null && cluster.pool != null) - cluster.pool.shutdown(); - } -} diff --git a/gi/posterior-regularisation/prjava/src/phrase/VB.java b/gi/posterior-regularisation/prjava/src/phrase/VB.java deleted file mode 100644 index cd3f4966..00000000 --- a/gi/posterior-regularisation/prjava/src/phrase/VB.java +++ /dev/null @@ -1,419 +0,0 @@ -package phrase;
-
-import gnu.trove.TIntArrayList;
-
-import io.FileUtil;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-import org.apache.commons.math.special.Gamma;
-
-import phrase.Corpus.Edge;
-
-public class VB {
-
- public static int MAX_ITER=400;
-
- /**@brief
- * hyper param for beta
- * where beta is multinomial
- * for generating words from a topic
- */
- public double lambda=0.1;
- /**@brief
- * hyper param for theta
- * where theta is dirichlet for z
- */
- public double alpha=0.0001;
- /**@brief
- * variational param for beta
- */
- private double rho[][][];
- private double digamma_rho[][][];
- private double rho_sum[][];
- /**@brief
- * variational param for z
- */
- //private double phi[][];
- /**@brief
- * variational param for theta
- */
- private double gamma[];
- private static double VAL_DIFF_RATIO=0.005;
-
- private int n_positions;
- private int n_words;
- private int K;
- private ExecutorService pool;
-
- private Corpus c;
- public static void main(String[] args) {
- // String in="../pdata/canned.con";
- String in="../pdata/btec.con";
- String out="../pdata/vb.out";
- int numCluster=25;
- Corpus corpus = null;
- File infile = new File(in);
- try {
- System.out.println("Reading concordance from " + infile);
- corpus = Corpus.readFromFile(FileUtil.reader(infile));
- corpus.printStats(System.out);
- } catch (IOException e) {
- System.err.println("Failed to open input file: " + infile);
- e.printStackTrace();
- System.exit(1);
- }
-
- VB vb=new VB(numCluster, corpus);
- int iter=20;
- for(int i=0;i<iter;i++){
- double obj=vb.EM();
- System.out.println("Iter "+i+": "+obj);
- }
-
- File outfile = new File (out);
- try {
- PrintStream ps = FileUtil.printstream(outfile);
- vb.displayPosterior(ps);
- // ps.println();
- // c2f.displayModelParam(ps);
- ps.close();
- } catch (IOException e) {
- System.err.println("Failed to open output file: " + outfile);
- e.printStackTrace();
- System.exit(1);
- }
- }
-
- public VB(int numCluster, Corpus corpus){
- c=corpus;
- K=numCluster;
- n_positions=c.getNumContextPositions();
- n_words=c.getNumWords();
- rho=new double[K][n_positions][n_words];
- //to init rho
- //loop through data and count up words
- double[] phi_tmp=new double[K];
- for(int i=0;i<K;i++){
- for(int pos=0;pos<n_positions;pos++){
- Arrays.fill(rho[i][pos], lambda);
- }
- }
- for(int d=0;d<c.getNumPhrases();d++){
- List<Edge>doc=c.getEdgesForPhrase(d);
- for(int n=0;n<doc.size();n++){
- TIntArrayList context=doc.get(n).getContext();
- arr.F.randomise(phi_tmp);
- for(int i=0;i<K;i++){
- for(int pos=0;pos<n_positions;pos++){
- rho[i][pos][context.get(pos)]+=phi_tmp[i];
- }
- }
- }
- }
-
- }
-
- private double inference(int phraseID, double[][] phi, double[] gamma)
- {
- List<Edge > doc=c.getEdgesForPhrase(phraseID);
- for(int i=0;i<phi.length;i++){
- for(int j=0;j<phi[i].length;j++){
- phi[i][j]=1.0/K;
- }
- }
- Arrays.fill(gamma,alpha+1.0/K);
-
- double digamma_gamma[]=new double[K];
-
- double gamma_sum=digamma(arr.F.l1norm(gamma));
- for(int i=0;i<K;i++){
- digamma_gamma[i]=digamma(gamma[i]);
- }
- double gammaSum[]=new double [K];
- double prev_val=0;
- double obj=0;
-
- for(int iter=0;iter<MAX_ITER;iter++){
- prev_val=obj;
- obj=0;
- Arrays.fill(gammaSum,0.0);
- for(int n=0;n<doc.size();n++){
- TIntArrayList context=doc.get(n).getContext();
- double phisum=0;
- for(int i=0;i<K;i++){
- double sum=0;
- for(int pos=0;pos<n_positions;pos++){
- int word=context.get(pos);
- sum+=digamma_rho[i][pos][word]-rho_sum[i][pos];
- }
- sum+= digamma_gamma[i]-gamma_sum;
- phi[n][i]=sum;
-
- if (i > 0){
- phisum = log_sum(phisum, phi[n][i]);
- }
- else{
- phisum = phi[n][i];
- }
-
- }//end of a word
-
- for(int i=0;i<K;i++){
- phi[n][i]=Math.exp(phi[n][i]-phisum);
- gammaSum[i]+=phi[n][i];
- }
-
- }//end of doc
-
- for(int i=0;i<K;i++){
- gamma[i]=alpha+gammaSum[i];
- }
- gamma_sum=digamma(arr.F.l1norm(gamma));
- for(int i=0;i<K;i++){
- digamma_gamma[i]=digamma(gamma[i]);
- }
- //compute objective for reporting
-
- obj=0;
-
- for(int i=0;i<K;i++){
- obj+=(alpha-1)*(digamma_gamma[i]-gamma_sum);
- }
-
-
- for(int n=0;n<doc.size();n++){
- TIntArrayList context=doc.get(n).getContext();
-
- for(int i=0;i<K;i++){
- //entropy of phi + expected log likelihood of z
- obj+=phi[n][i]*(digamma_gamma[i]-gamma_sum);
-
- if(phi[n][i]>1e-10){
- obj+=phi[n][i]*Math.log(phi[n][i]);
- }
-
- double beta_sum=0;
- for(int pos=0;pos<n_positions;pos++){
- int word=context.get(pos);
- beta_sum+=(digamma(rho[i][pos][word])-rho_sum[i][pos]);
- }
- obj+=phi[n][i]*beta_sum;
- }
- }
-
- obj-=log_gamma(arr.F.l1norm(gamma));
- for(int i=0;i<K;i++){
- obj+=Gamma.logGamma(gamma[i]);
- obj-=(gamma[i]-1)*(digamma_gamma[i]-gamma_sum);
- }
-
-// System.out.println(phraseID+": "+obj);
- if(iter>0 && (obj-prev_val)/Math.abs(obj)<VAL_DIFF_RATIO){
- break;
- }
- }//end of inference loop
-
- return obj;
- }//end of inference
-
- /**
- * @return objective of this iteration
- */
- public double EM(){
- double emObj=0;
- if(digamma_rho==null){
- digamma_rho=new double[K][n_positions][n_words];
- }
- for(int i=0;i<K;i++){
- for (int pos=0;pos<n_positions;pos++){
- for(int j=0;j<n_words;j++){
- digamma_rho[i][pos][j]= digamma(rho[i][pos][j]);
- }
- }
- }
-
- if(rho_sum==null){
- rho_sum=new double [K][n_positions];
- }
- for(int i=0;i<K;i++){
- for(int pos=0;pos<n_positions;pos++){
- rho_sum[i][pos]=digamma(arr.F.l1norm(rho[i][pos]));
- }
- }
-
- //E
- double exp_rho[][][]=new double[K][n_positions][n_words];
- if (pool == null)
- {
- for (int d=0;d<c.getNumPhrases();d++)
- {
- List<Edge > doc=c.getEdgesForPhrase(d);
- double[][] phi = new double[doc.size()][K];
- double[] gamma = new double[K];
-
- emObj += inference(d, phi, gamma);
-
- for(int n=0;n<doc.size();n++){
- TIntArrayList context=doc.get(n).getContext();
- for(int pos=0;pos<n_positions;pos++){
- int word=context.get(pos);
- for(int i=0;i<K;i++){
- exp_rho[i][pos][word]+=phi[n][i];
- }
- }
- }
- //if(d!=0 && d%100==0) System.out.print(".");
- //if(d!=0 && d%1000==0) System.out.println(d);
- }
- }
- else // multi-threaded version of above loop
- {
- class PartialEStep implements Callable<PartialEStep>
- {
- double[][] phi;
- double[] gamma;
- double obj;
- int d;
- PartialEStep(int d) { this.d = d; }
-
- public PartialEStep call()
- {
- phi = new double[c.getEdgesForPhrase(d).size()][K];
- gamma = new double[K];
- obj = inference(d, phi, gamma);
- return this;
- }
- }
-
- List<Future<PartialEStep>> jobs = new ArrayList<Future<PartialEStep>>();
- for (int d=0;d<c.getNumPhrases();d++)
- jobs.add(pool.submit(new PartialEStep(d)));
-
- for (Future<PartialEStep> job: jobs)
- {
- try {
- PartialEStep e = job.get();
-
- emObj += e.obj;
- List<Edge> doc = c.getEdgesForPhrase(e.d);
- for(int n=0;n<doc.size();n++){
- TIntArrayList context=doc.get(n).getContext();
- for(int pos=0;pos<n_positions;pos++){
- int word=context.get(pos);
- for(int i=0;i<K;i++){
- exp_rho[i][pos][word]+=e.phi[n][i];
- }
- }
- }
- } catch (ExecutionException e) {
- System.err.println("ERROR: E-step thread execution failed.");
- throw new RuntimeException(e);
- } catch (InterruptedException e) {
- System.err.println("ERROR: Failed to join E-step thread.");
- throw new RuntimeException(e);
- }
- }
- }
- // System.out.println("EM Objective:"+emObj);
-
- //M
- for(int i=0;i<K;i++){
- for(int pos=0;pos<n_positions;pos++){
- for(int j=0;j<n_words;j++){
- rho[i][pos][j]=lambda+exp_rho[i][pos][j];
- }
- }
- }
-
- //E[\log p(\beta|\lambda)] - E[\log q(\beta)]
- for(int i=0;i<K;i++){
- double rhoSum=0;
- for(int pos=0;pos<n_positions;pos++){
- for(int j=0;j<n_words;j++){
- rhoSum+=rho[i][pos][j];
- }
- double digamma_rhoSum=Gamma.digamma(rhoSum);
- emObj-=Gamma.logGamma(rhoSum);
- for(int j=0;j<n_words;j++){
- emObj+=(lambda-rho[i][pos][j])*(Gamma.digamma(rho[i][pos][j])-digamma_rhoSum);
- emObj+=Gamma.logGamma(rho[i][pos][j]);
- }
- }
- }
-
- return emObj;
- }//end of EM
-
- public void displayPosterior(PrintStream ps)
- {
- for(int d=0;d<c.getNumPhrases();d++){
- List<Edge > doc=c.getEdgesForPhrase(d);
- double[][] phi = new double[doc.size()][K];
- for(int i=0;i<phi.length;i++)
- for(int j=0;j<phi[i].length;j++)
- phi[i][j]=1.0/K;
- double[] gamma = new double[K];
-
- inference(d, phi, gamma);
-
- for(int n=0;n<doc.size();n++){
- Edge edge=doc.get(n);
- int tag=arr.F.argmax(phi[n]);
- ps.print(edge.getPhraseString());
- ps.print("\t");
- ps.print(edge.getContextString(true));
-
- ps.println(" ||| C=" + tag);
- }
- }
- }
-
- double log_sum(double log_a, double log_b)
- {
- double v;
-
- if (log_a < log_b)
- v = log_b+Math.log(1 + Math.exp(log_a-log_b));
- else
- v = log_a+Math.log(1 + Math.exp(log_b-log_a));
- return(v);
- }
-
- double digamma(double x)
- {
- double p;
- x=x+6;
- p=1/(x*x);
- p=(((0.004166666666667*p-0.003968253986254)*p+
- 0.008333333333333)*p-0.083333333333333)*p;
- p=p+Math.log(x)-0.5/x-1/(x-1)-1/(x-2)-1/(x-3)-1/(x-4)-1/(x-5)-1/(x-6);
- return p;
- }
-
- double log_gamma(double x)
- {
- double z=1/(x*x);
-
- x=x+6;
- z=(((-0.000595238095238*z+0.000793650793651)
- *z-0.002777777777778)*z+0.083333333333333)/x;
- z=(x-0.5)*Math.log(x)-x+0.918938533204673+z-Math.log(x-1)-
- Math.log(x-2)-Math.log(x-3)-Math.log(x-4)-Math.log(x-5)-Math.log(x-6);
- return z;
- }
-
- public void useThreadPool(ExecutorService threadPool)
- {
- pool = threadPool;
- }
-}//End of class
diff --git a/gi/posterior-regularisation/prjava/src/test/CorpusTest.java b/gi/posterior-regularisation/prjava/src/test/CorpusTest.java deleted file mode 100644 index b4c3041f..00000000 --- a/gi/posterior-regularisation/prjava/src/test/CorpusTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package test;
-
-import java.util.Arrays;
-import java.util.HashMap;
-
-import data.Corpus;
-import hmm.POS;
-
-public class CorpusTest {
-
- public static void main(String[] args) {
- Corpus c=new Corpus(POS.trainFilename);
-
-
- int idx=30;
-
-
- HashMap<String, Integer>vocab=
- (HashMap<String, Integer>) io.SerializedObjects.readSerializedObject(Corpus.alphaFilename);
-
- HashMap<String, Integer>tagVocab=
- (HashMap<String, Integer>) io.SerializedObjects.readSerializedObject(Corpus.tagalphaFilename);
-
-
- String [] dict=new String [vocab.size()+1];
- for(String key:vocab.keySet()){
- dict[vocab.get(key)]=key;
- }
- dict[dict.length-1]=Corpus.UNK_TOK;
-
- String [] tagdict=new String [tagVocab.size()+1];
- for(String key:tagVocab.keySet()){
- tagdict[tagVocab.get(key)]=key;
- }
- tagdict[tagdict.length-1]=Corpus.UNK_TOK;
-
- String[] sent=c.get(idx);
- int []data=c.getInt(idx);
-
-
- String []roundtrip=new String [sent.length];
- for(int i=0;i<sent.length;i++){
- roundtrip[i]=dict[data[i]];
- }
- System.out.println(Arrays.toString(sent));
- System.out.println(Arrays.toString(roundtrip));
-
- sent=c.tag.get(idx);
- data=c.tagData.get(idx);
-
-
- roundtrip=new String [sent.length];
- for(int i=0;i<sent.length;i++){
- roundtrip[i]=tagdict[data[i]];
- }
- System.out.println(Arrays.toString(sent));
- System.out.println(Arrays.toString(roundtrip));
- }
-
-}
diff --git a/gi/posterior-regularisation/prjava/src/test/HMMModelStats.java b/gi/posterior-regularisation/prjava/src/test/HMMModelStats.java deleted file mode 100644 index d54525c8..00000000 --- a/gi/posterior-regularisation/prjava/src/test/HMMModelStats.java +++ /dev/null @@ -1,105 +0,0 @@ -package test;
-
-import hmm.HMM;
-import hmm.POS;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-
-import data.Corpus;
-
-public class HMMModelStats {
-
- public static String modelFilename="../posdata/posModel.out";
- public static String alphaFilename="../posdata/corpus.alphabet";
- public static String statsFilename="../posdata/model.stats";
-
- public static final int NUM_WORD=50;
-
- public static String testFilename="../posdata/en_test.conll";
-
- public static double [][]maxwt;
-
- public static void main(String[] args) {
- HashMap<String, Integer>vocab=
- (HashMap<String, Integer>) io.SerializedObjects.readSerializedObject(alphaFilename);
-
- Corpus test=new Corpus(testFilename,vocab);
-
- String [] dict=new String [vocab.size()+1];
- for(String key:vocab.keySet()){
- dict[vocab.get(key)]=key;
- }
- dict[dict.length-1]=Corpus.UNK_TOK;
-
- HMM hmm=new HMM();
- hmm.readModel(modelFilename);
-
-
-
- PrintStream ps = null;
- try {
- ps = io.FileUtil.printstream(new File(statsFilename));
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(1);
- }
-
- double [][] emit=hmm.getEmitProb();
- for(int i=0;i<emit.length;i++){
- ArrayList<IntDoublePair>l=new ArrayList<IntDoublePair>();
- for(int j=0;j<emit[i].length;j++){
- l.add(new IntDoublePair(j,emit[i][j]));
- }
- Collections.sort(l);
- ps.println(i);
- for(int j=0;j<NUM_WORD;j++){
- if(j>=dict.length){
- break;
- }
- ps.print(dict[l.get(j).idx]+"\t");
- if((1+j)%10==0){
- ps.println();
- }
- }
- ps.println("\n");
- }
-
- checkMaxwt(hmm,ps,test.getAllData());
-
- int terminalSym=vocab.get(Corpus .END_SYM);
- //sample 10 sentences
- for(int i=0;i<10;i++){
- int []sent=hmm.sample(terminalSym);
- for(int j=0;j<sent.length;j++){
- ps.print(dict[sent[j]]+"\t");
- }
- ps.println();
- }
-
- ps.close();
-
- }
-
- public static void checkMaxwt(HMM hmm,PrintStream ps,int [][]data){
- double [][]emit=hmm.getEmitProb();
- maxwt=new double[emit.length][emit[0].length];
-
- hmm.computeMaxwt(maxwt,data);
- double sum=0;
- for(int i=0;i<maxwt.length;i++){
- for(int j=0;j<maxwt.length;j++){
- sum+=maxwt[i][j];
- }
- }
-
- ps.println("max w t P(w_i|t): "+sum);
-
- }
-
-}
diff --git a/gi/posterior-regularisation/prjava/src/test/IntDoublePair.java b/gi/posterior-regularisation/prjava/src/test/IntDoublePair.java deleted file mode 100644 index 3f9f0ad7..00000000 --- a/gi/posterior-regularisation/prjava/src/test/IntDoublePair.java +++ /dev/null @@ -1,23 +0,0 @@ -package test;
-
-public class IntDoublePair implements Comparable{
- double val;
- int idx;
- public int compareTo(Object o){
- if(o instanceof IntDoublePair){
- IntDoublePair pair=(IntDoublePair)o;
- if(pair.val>val){
- return 1;
- }
- if(pair.val<val){
- return -1;
- }
- return 0;
- }
- return -1;
- }
- public IntDoublePair(int i,double v){
- val=v;
- idx=i;
- }
-}
diff --git a/gi/posterior-regularisation/prjava/src/test/X2y2WithConstraints.java b/gi/posterior-regularisation/prjava/src/test/X2y2WithConstraints.java deleted file mode 100644 index 9059a59e..00000000 --- a/gi/posterior-regularisation/prjava/src/test/X2y2WithConstraints.java +++ /dev/null @@ -1,131 +0,0 @@ -package test;
-
-
-
-import optimization.gradientBasedMethods.ProjectedGradientDescent;
-import optimization.gradientBasedMethods.ProjectedObjective;
-import optimization.gradientBasedMethods.stats.OptimizerStats;
-import optimization.linesearch.ArmijoLineSearchMinimizationAlongProjectionArc;
-import optimization.linesearch.InterpolationPickFirstStep;
-import optimization.linesearch.LineSearchMethod;
-import optimization.projections.BoundsProjection;
-import optimization.projections.Projection;
-import optimization.projections.SimplexProjection;
-import optimization.stopCriteria.CompositeStopingCriteria;
-import optimization.stopCriteria.GradientL2Norm;
-import optimization.stopCriteria.ProjectedGradientL2Norm;
-import optimization.stopCriteria.StopingCriteria;
-import optimization.stopCriteria.ValueDifference;
-
-
-/**
- * @author javg
- *
- *
- *ax2+ b(y2 -displacement)
- */
-public class X2y2WithConstraints extends ProjectedObjective{
-
-
- double a, b;
- double dx;
- double dy;
- Projection projection;
-
-
- public X2y2WithConstraints(double a, double b, double[] params, double dx, double dy, Projection proj){
- //projection = new BoundsProjection(0.2,Double.MAX_VALUE);
- super();
- projection = proj;
- this.a = a;
- this.b = b;
- this.dx = dx;
- this.dy = dy;
- setInitialParameters(params);
- System.out.println("Function " +a+"(x-"+dx+")^2 + "+b+"(y-"+dy+")^2");
- System.out.println("Gradient " +(2*a)+"(x-"+dx+") ; "+(b*2)+"(y-"+dy+")");
- printParameters();
- projection.project(parameters);
- printParameters();
- gradient = new double[2];
- }
-
- public double getValue() {
- functionCalls++;
- return a*(parameters[0]-dx)*(parameters[0]-dx)+b*((parameters[1]-dy)*(parameters[1]-dy));
- }
-
- public double[] getGradient() {
- if(gradient == null){
- gradient = new double[2];
- }
- gradientCalls++;
- gradient[0]=2*a*(parameters[0]-dx);
- gradient[1]=2*b*(parameters[1]-dy);
- return gradient;
- }
-
-
- public double[] projectPoint(double[] point) {
- double[] newPoint = point.clone();
- projection.project(newPoint);
- return newPoint;
- }
-
- public void optimizeWithProjectedGradientDescent(LineSearchMethod ls, OptimizerStats stats, X2y2WithConstraints o){
- ProjectedGradientDescent optimizer = new ProjectedGradientDescent(ls);
- StopingCriteria stopGrad = new ProjectedGradientL2Norm(0.001);
- StopingCriteria stopValue = new ValueDifference(0.001);
- CompositeStopingCriteria compositeStop = new CompositeStopingCriteria();
- compositeStop.add(stopGrad);
- compositeStop.add(stopValue);
-
- optimizer.setMaxIterations(5);
- boolean succed = optimizer.optimize(o,stats,compositeStop);
- System.out.println("Ended optimzation Projected Gradient Descent\n" + stats.prettyPrint(1));
- System.out.println("Solution: " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1]);
- if(succed){
- System.out.println("Ended optimization in " + optimizer.getCurrentIteration());
- }else{
- System.out.println("Failed to optimize");
- }
- }
-
-
-
- public String toString(){
-
- return "P1: " + parameters[0] + " P2: " + parameters[1] + " value " + getValue() + " grad (" + getGradient()[0] + ":" + getGradient()[1]+")";
- }
-
- public static void main(String[] args) {
- double a = 1;
- double b=1;
- double x0 = 0;
- double y0 =1;
- double dx = 0.5;
- double dy = 0.2 ;
- double [] parameters = new double[2];
- parameters[0] = x0;
- parameters[1] = y0;
- X2y2WithConstraints o = new X2y2WithConstraints(a,b,parameters,dx,dy,
- new SimplexProjection(0.5)
- //new BoundsProjection(0.0,0.4)
- );
- System.out.println("Starting optimization " + " x0 " + o.parameters[0]+ " x1 " + o.parameters[1] + " a " + a + " b "+b );
- o.setDebugLevel(4);
-
- LineSearchMethod ls = new ArmijoLineSearchMinimizationAlongProjectionArc(new InterpolationPickFirstStep(1));
-
- OptimizerStats stats = new OptimizerStats();
- o.optimizeWithProjectedGradientDescent(ls, stats, o);
-
-// o = new x2y2WithConstraints(a,b,x0,y0,dx,dy);
-// stats = new OptimizerStats();
-// o.optimizeWithSpectralProjectedGradientDescent(stats, o);
- }
-
-
-
-
-}
diff --git a/gi/posterior-regularisation/prjava/src/util/Array.java b/gi/posterior-regularisation/prjava/src/util/Array.java deleted file mode 100644 index cc4725af..00000000 --- a/gi/posterior-regularisation/prjava/src/util/Array.java +++ /dev/null @@ -1,41 +0,0 @@ -package util; - -import java.util.Arrays; - -public class Array { - - - - public static void sortDescending(double[] ds){ - for (int i = 0; i < ds.length; i++) ds[i] = -ds[i]; - Arrays.sort(ds); - for (int i = 0; i < ds.length; i++) ds[i] = -ds[i]; - } - - /** - * Return a new reversed array - * @param array - * @return - */ - public static int[] reverseIntArray(int[] array){ - int[] reversed = new int[array.length]; - for (int i = 0; i < reversed.length; i++) { - reversed[i] = array[reversed.length-1-i]; - } - return reversed; - } - - public static String[] sumArray(String[] in, int from){ - String[] res = new String[in.length-from]; - for (int i = from; i < in.length; i++) { - res[i-from] = in[i]; - } - return res; - } - - public static void main(String[] args) { - int[] i = {1,2,3,4}; - util.Printing.printIntArray(i, null, "original"); - util.Printing.printIntArray(reverseIntArray(i), null, "reversed"); - } -} diff --git a/gi/posterior-regularisation/prjava/src/util/ArrayMath.java b/gi/posterior-regularisation/prjava/src/util/ArrayMath.java deleted file mode 100644 index 398a13a2..00000000 --- a/gi/posterior-regularisation/prjava/src/util/ArrayMath.java +++ /dev/null @@ -1,186 +0,0 @@ -package util; - -import java.util.Arrays; - -public class ArrayMath { - - public static double dotProduct(double[] v1, double[] v2) { - assert(v1.length == v2.length); - double result = 0; - for(int i = 0; i < v1.length; i++) - result += v1[i]*v2[i]; - return result; - } - - public static double twoNormSquared(double[] v) { - double result = 0; - for(double d : v) - result += d*d; - return result; - } - - public static boolean containsInvalid(double[] v) { - for(int i = 0; i < v.length; i++) - if(Double.isNaN(v[i]) || Double.isInfinite(v[i])) - return true; - return false; - } - - - - public static double safeAdd(double[] toAdd) { - // Make sure there are no positive infinities - double sum = 0; - for(int i = 0; i < toAdd.length; i++) { - assert(!(Double.isInfinite(toAdd[i]) && toAdd[i] > 0)); - assert(!Double.isNaN(toAdd[i])); - sum += toAdd[i]; - } - - return sum; - } - - /* Methods for filling integer and double arrays (of up to four dimensions) with the given value. */ - - public static void set(int[][][][] array, int value) { - for(int i = 0; i < array.length; i++) { - set(array[i], value); - } - } - - public static void set(int[][][] array, int value) { - for(int i = 0; i < array.length; i++) { - set(array[i], value); - } - } - - public static void set(int[][] array, int value) { - for(int i = 0; i < array.length; i++) { - set(array[i], value); - } - } - - public static void set(int[] array, int value) { - Arrays.fill(array, value); - } - - - public static void set(double[][][][] array, double value) { - for(int i = 0; i < array.length; i++) { - set(array[i], value); - } - } - - public static void set(double[][][] array, double value) { - for(int i = 0; i < array.length; i++) { - set(array[i], value); - } - } - - public static void set(double[][] array, double value) { - for(int i = 0; i < array.length; i++) { - set(array[i], value); - } - } - - public static void set(double[] array, double value) { - Arrays.fill(array, value); - } - - public static void setEqual(double[][][][] dest, double[][][][] source){ - for (int i = 0; i < source.length; i++) { - setEqual(dest[i],source[i]); - } - } - - - public static void setEqual(double[][][] dest, double[][][] source){ - for (int i = 0; i < source.length; i++) { - set(dest[i],source[i]); - } - } - - - public static void set(double[][] dest, double[][] source){ - for (int i = 0; i < source.length; i++) { - setEqual(dest[i],source[i]); - } - } - - public static void setEqual(double[] dest, double[] source){ - System.arraycopy(source, 0, dest, 0, source.length); - } - - public static void plusEquals(double[][][][] array, double val){ - for (int i = 0; i < array.length; i++) { - plusEquals(array[i], val); - } - } - - public static void plusEquals(double[][][] array, double val){ - for (int i = 0; i < array.length; i++) { - plusEquals(array[i], val); - } - } - - public static void plusEquals(double[][] array, double val){ - for (int i = 0; i < array.length; i++) { - plusEquals(array[i], val); - } - } - - public static void plusEquals(double[] array, double val){ - for (int i = 0; i < array.length; i++) { - array[i] += val; - } - } - - - public static double sum(double[] array) { - double res = 0; - for (int i = 0; i < array.length; i++) res += array[i]; - return res; - } - - - - public static double[][] deepclone(double[][] in){ - double[][] res = new double[in.length][]; - for (int i = 0; i < res.length; i++) { - res[i] = in[i].clone(); - } - return res; - } - - - public static double[][][] deepclone(double[][][] in){ - double[][][] res = new double[in.length][][]; - for (int i = 0; i < res.length; i++) { - res[i] = deepclone(in[i]); - } - return res; - } - - public static double cosine(double[] a, - double[] b) { - return (dotProduct(a, b)+1e-5)/(Math.sqrt(dotProduct(a, a)+1e-5)*Math.sqrt(dotProduct(b, b)+1e-5)); - } - - public static double max(double[] ds) { - double max = Double.NEGATIVE_INFINITY; - for(double d:ds) max = Math.max(d,max); - return max; - } - - public static void exponentiate(double[] a) { - for (int i = 0; i < a.length; i++) { - a[i] = Math.exp(a[i]); - } - } - - public static int sum(int[] array) { - int res = 0; - for (int i = 0; i < array.length; i++) res += array[i]; - return res; - } -} diff --git a/gi/posterior-regularisation/prjava/src/util/DifferentiableObjective.java b/gi/posterior-regularisation/prjava/src/util/DifferentiableObjective.java deleted file mode 100644 index 1ff1ae4a..00000000 --- a/gi/posterior-regularisation/prjava/src/util/DifferentiableObjective.java +++ /dev/null @@ -1,14 +0,0 @@ -package util; - -public interface DifferentiableObjective { - - public double getValue(); - - public void getGradient(double[] gradient); - - public void getParameters(double[] params); - - public void setParameters(double[] newParameters); - - public int getNumParameters(); -} diff --git a/gi/posterior-regularisation/prjava/src/util/DigammaFunction.java b/gi/posterior-regularisation/prjava/src/util/DigammaFunction.java deleted file mode 100644 index ff1478ad..00000000 --- a/gi/posterior-regularisation/prjava/src/util/DigammaFunction.java +++ /dev/null @@ -1,21 +0,0 @@ -package util; - -public class DigammaFunction { - public static double expDigamma(double number){ - if(number==0)return number; - return Math.exp(digamma(number)); - } - - public static double digamma(double number){ - if(number > 7){ - return digammApprox(number-0.5); - }else{ - return digamma(number+1) - 1.0/number; - } - } - - private static double digammApprox(double value){ - return Math.log(value) + 0.04167*Math.pow(value, -2) - 0.00729*Math.pow(value, -4) - + 0.00384*Math.pow(value, -6) - 0.00413*Math.pow(value, -8); - } -} diff --git a/gi/posterior-regularisation/prjava/src/util/FileSystem.java b/gi/posterior-regularisation/prjava/src/util/FileSystem.java deleted file mode 100644 index d7812e40..00000000 --- a/gi/posterior-regularisation/prjava/src/util/FileSystem.java +++ /dev/null @@ -1,21 +0,0 @@ -package util; - -import java.io.File; - -public class FileSystem { - public static boolean createDir(String directory) { - - File dir = new File(directory); - if (!dir.isDirectory()) { - boolean success = dir.mkdirs(); - if (!success) { - System.out.println("Unable to create directory " + directory); - return false; - } - System.out.println("Created directory " + directory); - } else { - System.out.println("Reusing directory " + directory); - } - return true; - } -} diff --git a/gi/posterior-regularisation/prjava/src/util/InputOutput.java b/gi/posterior-regularisation/prjava/src/util/InputOutput.java deleted file mode 100644 index da7f71bf..00000000 --- a/gi/posterior-regularisation/prjava/src/util/InputOutput.java +++ /dev/null @@ -1,67 +0,0 @@ -package util; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; -import java.util.Properties; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -public class InputOutput { - - /** - * Opens a file either compress with gzip or not compressed. - */ - public static BufferedReader openReader(String fileName) throws UnsupportedEncodingException, FileNotFoundException, IOException{ - System.out.println("Reading: " + fileName); - BufferedReader reader; - fileName = fileName.trim(); - if(fileName.endsWith("gz")){ - reader = new BufferedReader( - new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName)),"UTF8")); - }else{ - reader = new BufferedReader(new InputStreamReader( - new FileInputStream(fileName), "UTF8")); - } - - return reader; - } - - - public static PrintStream openWriter(String fileName) - throws UnsupportedEncodingException, FileNotFoundException, IOException{ - System.out.println("Writting to file: " + fileName); - PrintStream writter; - fileName = fileName.trim(); - if(fileName.endsWith("gz")){ - writter = new PrintStream(new GZIPOutputStream(new FileOutputStream(fileName)), - true, "UTF-8"); - - }else{ - writter = new PrintStream(new FileOutputStream(fileName), - true, "UTF-8"); - - } - - return writter; - } - - public static Properties readPropertiesFile(String fileName) { - Properties properties = new Properties(); - try { - properties.load(new FileInputStream(fileName)); - } catch (IOException e) { - e.printStackTrace(); - throw new AssertionError("Wrong properties file " + fileName); - } - System.out.println(properties.toString()); - - return properties; - } -} diff --git a/gi/posterior-regularisation/prjava/src/util/LogSummer.java b/gi/posterior-regularisation/prjava/src/util/LogSummer.java deleted file mode 100644 index 117393b9..00000000 --- a/gi/posterior-regularisation/prjava/src/util/LogSummer.java +++ /dev/null @@ -1,86 +0,0 @@ -package util; - -import java.lang.Math; - -/* - * Math tool for computing logs of sums, when the terms of the sum are already in log form. - * (Useful if the terms of the sum are very small numbers.) - */ -public class LogSummer { - - private LogSummer() { - } - - /** - * Given log(a) and log(b), computes log(a + b). - * - * @param loga log of first sum term - * @param logb log of second sum term - * @return log(sum), where sum = a + b - */ - public static double sum(double loga, double logb) { - assert(!Double.isNaN(loga)); - assert(!Double.isNaN(logb)); - - if(Double.isInfinite(loga)) - return logb; - if(Double.isInfinite(logb)) - return loga; - - double maxLog; - double difference; - if(loga > logb) { - difference = logb - loga; - maxLog = loga; - } - else { - difference = loga - logb; - maxLog = logb; - } - - return Math.log1p(Math.exp(difference)) + maxLog; - } - - /** - * Computes log(exp(array[index]) + b), and - * modifies array[index] to contain this new value. - * - * @param array array to modify - * @param index index at which to modify - * @param logb log of the second sum term - */ - public static void sum(double[] array, int index, double logb) { - array[index] = sum(array[index], logb); - } - - /** - * Computes log(a + b + c + ...) from log(a), log(b), log(c), ... - * by recursively splitting the input and delegating to the sum method. - * - * @param terms an array containing the log of all the terms for the sum - * @return log(sum), where sum = exp(terms[0]) + exp(terms[1]) + ... - */ - public static double sumAll(double... terms) { - return sumAllHelper(terms, 0, terms.length); - } - - /** - * Computes log(a_0 + a_1 + ...) from a_0 = exp(terms[begin]), - * a_1 = exp(terms[begin + 1]), ..., a_{end - 1 - begin} = exp(terms[end - 1]). - * - * @param terms an array containing the log of all the terms for the sum, - * and possibly some other terms that will not go into the sum - * @return log of the sum of the elements in the [begin, end) region of the terms array - */ - private static double sumAllHelper(final double[] terms, final int begin, final int end) { - int length = end - begin; - switch(length) { - case 0: return Double.NEGATIVE_INFINITY; - case 1: return terms[begin]; - default: - int midIndex = begin + length/2; - return sum(sumAllHelper(terms, begin, midIndex), sumAllHelper(terms, midIndex, end)); - } - } - -}
\ No newline at end of file diff --git a/gi/posterior-regularisation/prjava/src/util/MathUtil.java b/gi/posterior-regularisation/prjava/src/util/MathUtil.java deleted file mode 100644 index 799b1faf..00000000 --- a/gi/posterior-regularisation/prjava/src/util/MathUtil.java +++ /dev/null @@ -1,148 +0,0 @@ -package util; - -import java.util.Random; - -public class MathUtil { - public static final boolean closeToOne(double number){ - return Math.abs(number-1) < 1.E-10; - } - - public static final boolean closeToZero(double number){ - return Math.abs(number) < 1.E-5; - } - - /** - * Return a ramdom multinominal distribution. - * - * @param size - * @return - */ - public static final double[] randomVector(int size, Random r){ - double[] random = new double[size]; - double sum=0; - for(int i = 0; i < size; i++){ - double number = r.nextDouble(); - random[i] = number; - sum+=number; - } - for(int i = 0; i < size; i++){ - random[i] = random[i]/sum; - } - return random; - } - - - - public static double sum(double[] ds) { - double res = 0; - for (int i = 0; i < ds.length; i++) { - res+=ds[i]; - } - return res; - } - - public static double max(double[] ds) { - double res = Double.NEGATIVE_INFINITY; - for (int i = 0; i < ds.length; i++) { - res = Math.max(res, ds[i]); - } - return res; - } - - public static double min(double[] ds) { - double res = Double.POSITIVE_INFINITY; - for (int i = 0; i < ds.length; i++) { - res = Math.min(res, ds[i]); - } - return res; - } - - - public static double KLDistance(double[] p, double[] q) { - int len = p.length; - double kl = 0; - for (int j = 0; j < len; j++) { - if (p[j] == 0 || q[j] == 0) { - continue; - } else { - kl += q[j] * Math.log(q[j] / p[j]); - } - - } - return kl; - } - - public static double L2Distance(double[] p, double[] q) { - int len = p.length; - double l2 = 0; - for (int j = 0; j < len; j++) { - if (p[j] == 0 || q[j] == 0) { - continue; - } else { - l2 += (q[j] - p[j])*(q[j] - p[j]); - } - - } - return Math.sqrt(l2); - } - - public static double L1Distance(double[] p, double[] q) { - int len = p.length; - double l1 = 0; - for (int j = 0; j < len; j++) { - if (p[j] == 0 || q[j] == 0) { - continue; - } else { - l1 += Math.abs(q[j] - p[j]); - } - - } - return l1; - } - - public static double dot(double[] ds, double[] ds2) { - double res = 0; - for (int i = 0; i < ds2.length; i++) { - res+= ds[i]*ds2[i]; - } - return res; - } - - public static double expDigamma(double number){ - return Math.exp(digamma(number)); - } - - public static double digamma(double number){ - if(number > 7){ - return digammApprox(number-0.5); - }else{ - return digamma(number+1) - 1.0/number; - } - } - - private static double digammApprox(double value){ - return Math.log(value) + 0.04167*Math.pow(value, -2) - 0.00729*Math.pow(value, -4) - + 0.00384*Math.pow(value, -6) - 0.00413*Math.pow(value, -8); - } - - public static double eulerGamma = 0.57721566490152386060651209008240243; - // FIXME -- so far just the initialization from Minka's paper "Estimating a Dirichlet distribution". - public static double invDigamma(double y) { - if (y>= -2.22) return Math.exp(y)+0.5; - return -1.0/(y+eulerGamma); - } - - - - public static void main(String[] args) { - for(double i = 0; i < 10 ; i+=0.1){ - System.out.println(i+"\t"+expDigamma(i)+"\t"+(i-0.5)); - } -// double gammaValue = (expDigamma(3)/expDigamma(10) + expDigamma(3)/expDigamma(10) + expDigamma(4)/expDigamma(10)); -// double normalValue = 3/10+3/4+10/10; -// System.out.println("Gamma " + gammaValue + " normal " + normalValue); - } - - - -} diff --git a/gi/posterior-regularisation/prjava/src/util/Matrix.java b/gi/posterior-regularisation/prjava/src/util/Matrix.java deleted file mode 100644 index 8fb6d911..00000000 --- a/gi/posterior-regularisation/prjava/src/util/Matrix.java +++ /dev/null @@ -1,16 +0,0 @@ -package util; - -public class Matrix { - int x; - int y; - double[][] values; - - public Matrix(int x, int y){ - this.x = x; - this.y=y; - values = new double[x][y]; - } - - - -} diff --git a/gi/posterior-regularisation/prjava/src/util/MemoryTracker.java b/gi/posterior-regularisation/prjava/src/util/MemoryTracker.java deleted file mode 100644 index 83a65611..00000000 --- a/gi/posterior-regularisation/prjava/src/util/MemoryTracker.java +++ /dev/null @@ -1,47 +0,0 @@ -package util; - - -public class MemoryTracker { - - double initM,finalM; - boolean start = false,finish = false; - - public MemoryTracker(){ - - } - - public void start(){ - System.gc(); - System.gc(); - System.gc(); - initM = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/(1024*1024); - start = true; - } - - public void finish(){ - if(!start){ - throw new RuntimeException("Canot stop before starting"); - } - System.gc(); - System.gc(); - System.gc(); - finalM = (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/(1024*1024); - finish = true; - } - - public String print(){ - if(!finish){ - throw new RuntimeException("Canot print before stopping"); - } - return "Used: " + (finalM - initM) + "MB"; - } - - public void clear(){ - initM = 0; - finalM = 0; - finish = false; - start = false; - } - - -} diff --git a/gi/posterior-regularisation/prjava/src/util/Pair.java b/gi/posterior-regularisation/prjava/src/util/Pair.java deleted file mode 100644 index 7b1f108d..00000000 --- a/gi/posterior-regularisation/prjava/src/util/Pair.java +++ /dev/null @@ -1,31 +0,0 @@ -package util; - -public class Pair<O1, O2> { - public O1 _first; - public O2 _second; - - public final O1 first() { - return _first; - } - - public final O2 second() { - return _second; - } - - public final void setFirst(O1 value){ - _first = value; - } - - public final void setSecond(O2 value){ - _second = value; - } - - public Pair(O1 first, O2 second) { - _first = first; - _second = second; - } - - public String toString(){ - return _first + " " + _second; - } -} diff --git a/gi/posterior-regularisation/prjava/src/util/Printing.java b/gi/posterior-regularisation/prjava/src/util/Printing.java deleted file mode 100644 index 14fcbe91..00000000 --- a/gi/posterior-regularisation/prjava/src/util/Printing.java +++ /dev/null @@ -1,158 +0,0 @@ -package util; - -public class Printing { - static java.text.DecimalFormat fmt = new java.text.DecimalFormat(); - - public static String padWithSpace(String s, int len){ - StringBuffer sb = new StringBuffer(); - while(sb.length() +s.length() < len){ - sb.append(" "); - } - sb.append(s); - return sb.toString(); - } - - public static String prettyPrint(double d, String patt, int len) { - fmt.applyPattern(patt); - String s = fmt.format(d); - while (s.length() < len) { - s = " " + s; - } - return s; - } - - public static String formatTime(long duration) { - StringBuilder sb = new StringBuilder(); - double d = duration / 1000; - fmt.applyPattern("00"); - sb.append(fmt.format((int) (d / (60 * 60))) + ":"); - d -= ((int) d / (60 * 60)) * 60 * 60; - sb.append(fmt.format((int) (d / 60)) + ":"); - d -= ((int) d / 60) * 60; - fmt.applyPattern("00.0"); - sb.append(fmt.format(d)); - return sb.toString(); - } - - - public static String doubleArrayToString(double[] array, String[] labels, String arrayName) { - StringBuffer res = new StringBuffer(); - res.append(arrayName); - res.append("\n"); - for (int i = 0; i < array.length; i++) { - if (labels == null){ - res.append(i+" \t"); - }else{ - res.append(labels[i]+ "\t"); - } - } - res.append("sum\n"); - double sum = 0; - for (int i = 0; i < array.length; i++) { - res.append(prettyPrint(array[i], - "0.00000E00", 8) + "\t"); - sum+=array[i]; - } - res.append(prettyPrint(sum, - "0.00000E00", 8)+"\n"); - return res.toString(); - } - - - - public static void printDoubleArray(double[] array, String labels[], String arrayName) { - System.out.println(doubleArrayToString(array, labels,arrayName)); - } - - - public static String doubleArrayToString(double[][] array, String[] labels1, String[] labels2, - String arrayName){ - StringBuffer res = new StringBuffer(); - res.append(arrayName); - res.append("\n\t"); - //Calculates the column sum to keeps the sums - double[] sums = new double[array[0].length+1]; - //Prints rows headings - for (int i = 0; i < array[0].length; i++) { - if (labels1 == null){ - res.append(i+" \t"); - }else{ - res.append(labels1[i]+" \t"); - } - } - res.append("sum\n"); - double sum = 0; - //For each row print heading - for (int i = 0; i < array.length; i++) { - if (labels2 == null){ - res.append(i+"\t"); - }else{ - res.append(labels2[i]+"\t"); - } - //Print values for that row - for (int j = 0; j < array[0].length; j++) { - res.append(" " + prettyPrint(array[i][j], - "0.00000E00", 8) + "\t"); - sums[j] += array[i][j]; - sum+=array[i][j]; //Sum all values of that row - } - //Print row sum - res.append(prettyPrint(sum,"0.00000E00", 8)+"\n"); - sums[array[0].length]+=sum; - sum=0; - } - res.append("sum\t"); - //Print values for colums sum - for (int i = 0; i < array[0].length+1; i++) { - res.append(prettyPrint(sums[i],"0.00000E00", 8)+"\t"); - } - res.append("\n"); - return res.toString(); - } - - public static void printDoubleArray(double[][] array, String[] labels1, String[] labels2 - , String arrayName) { - System.out.println(doubleArrayToString(array, labels1,labels2,arrayName)); - } - - - public static void printIntArray(int[][] array, String[] labels1, String[] labels2, String arrayName, - int size1, int size2) { - System.out.println(arrayName); - for (int i = 0; i < size1; i++) { - for (int j = 0; j < size2; j++) { - System.out.print(" " + array[i][j] + " "); - - } - System.out.println(); - } - System.out.println(); - } - - public static String intArrayToString(int[] array, String[] labels, String arrayName) { - StringBuffer res = new StringBuffer(); - res.append(arrayName); - for (int i = 0; i < array.length; i++) { - res.append(" " + array[i] + " "); - - } - res.append("\n"); - return res.toString(); - } - - public static void printIntArray(int[] array, String[] labels, String arrayName) { - System.out.println(intArrayToString(array, labels,arrayName)); - } - - public static String toString(double[][] d){ - StringBuffer sb = new StringBuffer(); - for (int i = 0; i < d.length; i++) { - for (int j = 0; j < d[0].length; j++) { - sb.append(prettyPrint(d[i][j], "0.00E0", 10)); - } - sb.append("\n"); - } - return sb.toString(); - } - -} diff --git a/gi/posterior-regularisation/prjava/src/util/Sorters.java b/gi/posterior-regularisation/prjava/src/util/Sorters.java deleted file mode 100644 index 836444e5..00000000 --- a/gi/posterior-regularisation/prjava/src/util/Sorters.java +++ /dev/null @@ -1,39 +0,0 @@ -package util; - -import java.util.Comparator; - -public class Sorters { - public static class sortWordsCounts implements Comparator{ - - /** - * Sorter for a pair of word id, counts. Sort ascending by counts - */ - public int compare(Object arg0, Object arg1) { - Pair<Integer,Integer> p1 = (Pair<Integer,Integer>)arg0; - Pair<Integer,Integer> p2 = (Pair<Integer,Integer>)arg1; - if(p1.second() > p2.second()){ - return 1; - }else{ - return -1; - } - } - - } - -public static class sortWordsDouble implements Comparator{ - - /** - * Sorter for a pair of word id, counts. Sort by counts - */ - public int compare(Object arg0, Object arg1) { - Pair<Integer,Double> p1 = (Pair<Integer,Double>)arg0; - Pair<Integer,Double> p2 = (Pair<Integer,Double>)arg1; - if(p1.second() < p2.second()){ - return 1; - }else{ - return -1; - } - } - - } -} diff --git a/gi/posterior-regularisation/prjava/train-PR-cluster.sh b/gi/posterior-regularisation/prjava/train-PR-cluster.sh deleted file mode 100755 index 67552c00..00000000 --- a/gi/posterior-regularisation/prjava/train-PR-cluster.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -d=`dirname $0` -java -ea -Xmx30g -cp $d/prjava.jar:$d/lib/trove-2.0.2.jar:$d/lib/optimization.jar:$d/lib/jopt-simple-3.2.jar:$d/lib/commons-math-2.1.jar phrase.Trainer $* diff --git a/gi/posterior-regularisation/projected_gradient.cc b/gi/posterior-regularisation/projected_gradient.cc deleted file mode 100644 index f7c39817..00000000 --- a/gi/posterior-regularisation/projected_gradient.cc +++ /dev/null @@ -1,87 +0,0 @@ -// -// Minimises given functional using the projected gradient method. Based on -// algorithm and demonstration example in Linear and Nonlinear Programming, -// Luenberger and Ye, 3rd ed., p 370. -// - -#include "invert.hh" -#include <iostream> - -using namespace std; - -double -f(double x1, double x2, double x3, double x4) -{ - return x1 * x1 + x2 * x2 + x3 * x3 + x4 * x4 - 2 * x1 - 3 * x4; -} - -ublas::vector<double> -g(double x1, double x2, double x3, double x4) -{ - ublas::vector<double> v(4); - v(0) = 2 * x1 - 2; - v(1) = 2 * x2; - v(2) = 2 * x3; - v(3) = 2 * x4 - 3; - return v; -} - -ublas::matrix<double> -activeConstraints(double x1, double x2, double x3, double x4) -{ - int n = 2; - if (x1 == 0) ++n; - if (x2 == 0) ++n; - if (x3 == 0) ++n; - if (x4 == 0) ++n; - - ublas::matrix<double> a(n,4); - a(0, 0) = 2; a(0, 1) = 1; a(0, 2) = 1; a(0, 3) = 4; - a(1, 0) = 1; a(1, 1) = 1; a(1, 2) = 2; a(1, 3) = 1; - - int c = 2; - if (x1 == 0) a(c++, 0) = 1; - if (x2 == 0) a(c++, 1) = 1; - if (x3 == 0) a(c++, 2) = 1; - if (x4 == 0) a(c++, 3) = 1; - - return a; -} - -ublas::matrix<double> -projection(const ublas::matrix<double> &a) -{ - ublas::matrix<double> aT = ublas::trans(a); - ublas::matrix<double> inv(a.size1(), a.size1()); - bool ok = invert_matrix(ublas::matrix<double>(ublas::prod(a, aT)), inv); - assert(ok && "Failed to invert matrix"); - return ublas::identity_matrix<double>(4) - - ublas::prod(aT, ublas::matrix<double>(ublas::prod(inv, a))); -} - -int main(int argc, char *argv[]) -{ - double x1 = 2, x2 = 2, x3 = 1, x4 = 0; - - double fval = f(x1, x2, x3, x4); - cout << "f = " << fval << endl; - ublas::vector<double> grad = g(x1, x2, x3, x4); - cout << "g = " << grad << endl; - ublas::matrix<double> A = activeConstraints(x1, x2, x3, x4); - cout << "A = " << A << endl; - ublas::matrix<double> P = projection(A); - cout << "P = " << P << endl; - // the direction of movement - ublas::vector<double> d = prod(P, grad); - cout << "d = " << (d / d(0)) << endl; - - // special case for d = 0 - - // next solve for limits on the line search - - // then use golden rule technique between these values (if bounded) - - // or simple Armijo's rule technique - - return 0; -} diff --git a/gi/posterior-regularisation/simplex_pg.py b/gi/posterior-regularisation/simplex_pg.py deleted file mode 100644 index 5da796d3..00000000 --- a/gi/posterior-regularisation/simplex_pg.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Following Leunberger and Ye, Linear and Nonlinear Progamming, 3rd ed. p367 -# "The gradient projection method" -# applied to an equality constraint for a simplex. -# -# min f(x) -# s.t. x >= 0, sum_i x = d -# -# FIXME: enforce the positivity constraint - a limit on the line search? -# - -from numpy import * -from scipy import * -from linesearch import line_search -# local copy of scipy's Amijo line_search - wasn't enforcing alpha max correctly -import sys - -dims = 4 - -def f(x): - fv = x[0]*x[0] + x[1]*x[1] + x[2]*x[2] + x[3]*x[3] - 2*x[0] - 3*x[3] - # print 'evaluating f at', x, 'value', fv - return fv - -def g(x): - return array([2*x[0] - 2, 2*x[1], 2*x[2], 2*x[3]-3]) - -def pg(x): - gv = g(x) - return gv - sum(gv) / dims - -x = ones(dims) / dims -old_fval = None - -while True: - fv = f(x) - gv = g(x) - dv = pg(x) - - print 'x', x, 'f', fv, 'g', gv, 'd', dv - - if old_fval == None: - old_fval = fv + 0.1 - - # solve for maximum step size i.e. when positivity constraints kick in - # x - alpha d = 0 => alpha = x/d - amax = max(x/dv) - if amax < 1e-8: break - - stuff = line_search(f, pg, x, -dv, dv, fv, old_fval, amax=amax) - alpha = stuff[0] # Nb. can avoid next evaluation of f,g,d using 'stuff' - if alpha < 1e-8: break - x -= alpha * dv - - old_fval = fv diff --git a/gi/posterior-regularisation/split-languages.py b/gi/posterior-regularisation/split-languages.py deleted file mode 100755 index 206da661..00000000 --- a/gi/posterior-regularisation/split-languages.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python - -import sys - -sout = open(sys.argv[1], 'w') -tout = open(sys.argv[2], 'w') -for line in sys.stdin: - phrase, contexts = line.rstrip().split('\t') - sp, tp = phrase.split(' <SPLIT> ') - sout.write('%s\t' % sp) - tout.write('%s\t' % tp) - parts = contexts.split(' ||| ') - for i in range(0, len(parts), 2): - sc, tc = parts[i].split(' <SPLIT> ') - if i != 0: - sout.write(' ||| ') - tout.write(' ||| ') - sout.write('%s ||| %s' % (sc, parts[i+1])) - tout.write('%s ||| %s' % (tc, parts[i+1])) - sout.write('\n') - tout.write('\n') -sout.close() -tout.close() diff --git a/gi/posterior-regularisation/train_pr_agree.py b/gi/posterior-regularisation/train_pr_agree.py deleted file mode 100644 index 9d41362d..00000000 --- a/gi/posterior-regularisation/train_pr_agree.py +++ /dev/null @@ -1,400 +0,0 @@ -import sys -import scipy.optimize -from scipy.stats import geom -from numpy import * -from numpy.random import random, seed - -style = sys.argv[1] -if len(sys.argv) >= 3: - seed(int(sys.argv[2])) - -# -# Step 1: load the concordance counts -# - -edges = [] -word_types = {} -phrase_types = {} -context_types = {} - -for line in sys.stdin: - phrase, rest = line.strip().split('\t') - ptoks = tuple(map(lambda t: word_types.setdefault(t, len(word_types)), phrase.split())) - pid = phrase_types.setdefault(ptoks, len(phrase_types)) - - parts = rest.split('|||') - for i in range(0, len(parts), 2): - context, count = parts[i:i+2] - - ctx = filter(lambda x: x != '<PHRASE>', context.split()) - ctoks = tuple(map(lambda t: word_types.setdefault(t, len(word_types)), ctx)) - cid = context_types.setdefault(ctoks, len(context_types)) - - cnt = int(count.strip()[2:]) - edges.append((pid, cid, cnt)) - -word_type_list = [None] * len(word_types) -for typ, index in word_types.items(): - word_type_list[index] = typ - -phrase_type_list = [None] * len(phrase_types) -for typ, index in phrase_types.items(): - phrase_type_list[index] = typ - -context_type_list = [None] * len(context_types) -for typ, index in context_types.items(): - context_type_list[index] = typ - -num_tags = 5 -num_types = len(word_types) -num_phrases = len(phrase_types) -num_contexts = len(context_types) -num_edges = len(edges) - -print 'Read in', num_edges, 'edges', num_phrases, 'phrases', num_contexts, 'contexts and', num_types, 'word types' - -# -# Step 2: expectation maximisation -# - -def normalise(a): - return a / float(sum(a)) - -class PhraseToContextModel: - def __init__(self): - # Pr(tag | phrase) - self.tagDist = [normalise(random(num_tags)+1) for p in range(num_phrases)] - # Pr(context at pos i = w | tag) indexed by i, tag, word - self.contextWordDist = [[normalise(random(num_types)+1) for t in range(num_tags)] for i in range(4)] - - def prob(self, pid, cid): - # return distribution p(tag, context | phrase) as vector of length |tags| - context = context_type_list[cid] - dist = zeros(num_tags) - for t in range(num_tags): - prob = self.tagDist[pid][t] - for k, tokid in enumerate(context): - prob *= self.contextWordDist[k][t][tokid] - dist[t] = prob - return dist - - def expectation_maximisation_step(self, lamba=None): - tagCounts = zeros((num_phrases, num_tags)) - contextWordCounts = zeros((4, num_tags, num_types)) - - # E-step - llh = 0 - for pid, cid, cnt in edges: - q = self.prob(pid, cid) - z = sum(q) - q /= z - llh += log(z) - context = context_type_list[cid] - if lamba != None: - q *= exp(lamba) - q /= sum(q) - for t in range(num_tags): - tagCounts[pid][t] += cnt * q[t] - for i in range(4): - for t in range(num_tags): - contextWordCounts[i][t][context[i]] += cnt * q[t] - - # M-step - for p in range(num_phrases): - self.tagDist[p] = normalise(tagCounts[p]) - for i in range(4): - for t in range(num_tags): - self.contextWordDist[i][t] = normalise(contextWordCounts[i,t]) - - return llh - -class ContextToPhraseModel: - def __init__(self): - # Pr(tag | context) = Multinomial - self.tagDist = [normalise(random(num_tags)+1) for p in range(num_contexts)] - # Pr(phrase = w | tag) = Multinomial - self.phraseSingleDist = [normalise(random(num_types)+1) for t in range(num_tags)] - # Pr(phrase_1 = w | tag) = Multinomial - self.phraseLeftDist = [normalise(random(num_types)+1) for t in range(num_tags)] - # Pr(phrase_-1 = w | tag) = Multinomial - self.phraseRightDist = [normalise(random(num_types)+1) for t in range(num_tags)] - # Pr(|phrase| = l | tag) = Geometric - self.phraseLengthDist = [0.5] * num_tags - # n.b. internal words for phrases of length >= 3 are drawn from uniform distribution - - def prob(self, pid, cid): - # return distribution p(tag, phrase | context) as vector of length |tags| - phrase = phrase_type_list[pid] - dist = zeros(num_tags) - for t in range(num_tags): - prob = self.tagDist[cid][t] - f = self.phraseLengthDist[t] - prob *= geom.pmf(len(phrase), f) - if len(phrase) == 1: - prob *= self.phraseSingleDist[t][phrase[0]] - else: - prob *= self.phraseLeftDist[t][phrase[0]] - prob *= self.phraseRightDist[t][phrase[-1]] - dist[t] = prob - return dist - - def expectation_maximisation_step(self, lamba=None): - tagCounts = zeros((num_contexts, num_tags)) - phraseSingleCounts = zeros((num_tags, num_types)) - phraseLeftCounts = zeros((num_tags, num_types)) - phraseRightCounts = zeros((num_tags, num_types)) - phraseLength = zeros(num_types) - - # E-step - llh = 0 - for pid, cid, cnt in edges: - q = self.prob(pid, cid) - z = sum(q) - q /= z - llh += log(z) - if lamba != None: - q *= exp(lamba) - q /= sum(q) - #print 'p', phrase_type_list[pid], 'c', context_type_list[cid], 'q', q - phrase = phrase_type_list[pid] - for t in range(num_tags): - tagCounts[cid][t] += cnt * q[t] - phraseLength[t] += cnt * len(phrase) * q[t] - if len(phrase) == 1: - phraseSingleCounts[t][phrase[0]] += cnt * q[t] - else: - phraseLeftCounts[t][phrase[0]] += cnt * q[t] - phraseRightCounts[t][phrase[-1]] += cnt * q[t] - - # M-step - for t in range(num_tags): - self.phraseLengthDist[t] = min(max(sum(tagCounts[:,t]) / phraseLength[t], 1e-6), 1-1e-6) - self.phraseSingleDist[t] = normalise(phraseSingleCounts[t]) - self.phraseLeftDist[t] = normalise(phraseLeftCounts[t]) - self.phraseRightDist[t] = normalise(phraseRightCounts[t]) - for c in range(num_contexts): - self.tagDist[c] = normalise(tagCounts[c]) - - #print 't', self.tagDist - #print 'l', self.phraseLengthDist - #print 's', self.phraseSingleDist - #print 'L', self.phraseLeftDist - #print 'R', self.phraseRightDist - - return llh - -class ProductModel: - """ - WARNING: I haven't verified the maths behind this model. It's quite likely to be incorrect. - """ - - def __init__(self): - self.pcm = PhraseToContextModel() - self.cpm = ContextToPhraseModel() - - def prob(self, pid, cid): - p1 = self.pcm.prob(pid, cid) - p2 = self.cpm.prob(pid, cid) - return (p1 / sum(p1)) * (p2 / sum(p2)) - - def expectation_maximisation_step(self): - tagCountsGivenPhrase = zeros((num_phrases, num_tags)) - contextWordCounts = zeros((4, num_tags, num_types)) - - tagCountsGivenContext = zeros((num_contexts, num_tags)) - phraseSingleCounts = zeros((num_tags, num_types)) - phraseLeftCounts = zeros((num_tags, num_types)) - phraseRightCounts = zeros((num_tags, num_types)) - phraseLength = zeros(num_types) - - kl = llh1 = llh2 = 0 - for pid, cid, cnt in edges: - p1 = self.pcm.prob(pid, cid) - llh1 += log(sum(p1)) * cnt - p2 = self.cpm.prob(pid, cid) - llh2 += log(sum(p2)) * cnt - - q = (p1 / sum(p1)) * (p2 / sum(p2)) - kl += log(sum(q)) * cnt - qi = sqrt(q) - qi /= sum(qi) - - phrase = phrase_type_list[pid] - context = context_type_list[cid] - for t in range(num_tags): - tagCountsGivenPhrase[pid][t] += cnt * qi[t] - tagCountsGivenContext[cid][t] += cnt * qi[t] - phraseLength[t] += cnt * len(phrase) * qi[t] - if len(phrase) == 1: - phraseSingleCounts[t][phrase[0]] += cnt * qi[t] - else: - phraseLeftCounts[t][phrase[0]] += cnt * qi[t] - phraseRightCounts[t][phrase[-1]] += cnt * qi[t] - for i in range(4): - contextWordCounts[i][t][context[i]] += cnt * qi[t] - - kl *= -2 - - for t in range(num_tags): - for i in range(4): - self.pcm.contextWordDist[i][t] = normalise(contextWordCounts[i,t]) - self.cpm.phraseLengthDist[t] = min(max(sum(tagCountsGivenContext[:,t]) / phraseLength[t], 1e-6), 1-1e-6) - self.cpm.phraseSingleDist[t] = normalise(phraseSingleCounts[t]) - self.cpm.phraseLeftDist[t] = normalise(phraseLeftCounts[t]) - self.cpm.phraseRightDist[t] = normalise(phraseRightCounts[t]) - for p in range(num_phrases): - self.pcm.tagDist[p] = normalise(tagCountsGivenPhrase[p]) - for c in range(num_contexts): - self.cpm.tagDist[c] = normalise(tagCountsGivenContext[c]) - - # return the overall objective - return llh1 + llh2 + kl - -class RegularisedProductModel: - # as above, but with a slack regularisation term which kills the - # closed-form solution for the E-step - - def __init__(self, epsilon): - self.pcm = PhraseToContextModel() - self.cpm = ContextToPhraseModel() - self.epsilon = epsilon - self.lamba = zeros(num_tags) - - def prob(self, pid, cid): - p1 = self.pcm.prob(pid, cid) - p2 = self.cpm.prob(pid, cid) - return (p1 / sum(p1)) * (p2 / sum(p2)) - - def dual(self, lamba): - return self.logz(lamba) + self.epsilon * dot(lamba, lamba) ** 0.5 - - def dual_gradient(self, lamba): - return self.expected_features(lamba) + self.epsilon * 2 * lamba - - def expectation_maximisation_step(self): - # PR-step: optimise lambda to minimise log(z_lambda) + eps ||lambda||_2 - self.lamba = scipy.optimize.fmin_slsqp(self.dual, self.lamba, - bounds=[(0, 1e100)] * num_tags, - fprime=self.dual_gradient, iprint=1) - - # E,M-steps: collect expected counts under q_lambda and normalise - llh1 = self.pcm.expectation_maximisation_step(self.lamba) - llh2 = self.cpm.expectation_maximisation_step(-self.lamba) - - # return the overall objective: llh - KL(q||p1.p2) - # llh = llh1 + llh2 - # kl = sum q log q / p1 p2 = sum q { lambda . phi } - log Z - return llh1 + llh2 + self.logz(self.lamba) \ - - dot(self.lamba, self.expected_features(self.lamba)) - - def logz(self, lamba): - lz = 0 - for pid, cid, cnt in edges: - p1 = self.pcm.prob(pid, cid) - z1 = dot(p1 / sum(p1), exp(lamba)) - lz += log(z1) * cnt - - p2 = self.cpm.prob(pid, cid) - z2 = dot(p2 / sum(p2), exp(-lamba)) - lz += log(z2) * cnt - return lz - - def expected_features(self, lamba): - fs = zeros(num_tags) - for pid, cid, cnt in edges: - p1 = self.pcm.prob(pid, cid) - q1 = (p1 / sum(p1)) * exp(lamba) - fs += cnt * q1 / sum(q1) - - p2 = self.cpm.prob(pid, cid) - q2 = (p2 / sum(p2)) * exp(-lamba) - fs -= cnt * q2 / sum(q2) - return fs - - -class InterpolatedModel: - def __init__(self, epsilon): - self.pcm = PhraseToContextModel() - self.cpm = ContextToPhraseModel() - self.epsilon = epsilon - self.lamba = zeros(num_tags) - - def prob(self, pid, cid): - p1 = self.pcm.prob(pid, cid) - p2 = self.cpm.prob(pid, cid) - return (p1 + p2) / 2 - - def dual(self, lamba): - return self.logz(lamba) + self.epsilon * dot(lamba, lamba) ** 0.5 - - def dual_gradient(self, lamba): - return self.expected_features(lamba) + self.epsilon * 2 * lamba - - def expectation_maximisation_step(self): - # PR-step: optimise lambda to minimise log(z_lambda) + eps ||lambda||_2 - self.lamba = scipy.optimize.fmin_slsqp(self.dual, self.lamba, - bounds=[(0, 1e100)] * num_tags, - fprime=self.dual_gradient, iprint=2) - - # E,M-steps: collect expected counts under q_lambda and normalise - llh1 = self.pcm.expectation_maximisation_step(self.lamba) - llh2 = self.cpm.expectation_maximisation_step(self.lamba) - - # return the overall objective: llh1 + llh2 - KL(q||p1.p2) - # kl = sum_y q log q / 0.5 * (p1 + p2) = sum_y q(y) { -lambda . phi(y) } - log Z - # = -log Z + lambda . (E_q1[-phi] + E_q2[-phi]) / 2 - kl = -self.logz(self.lamba) + dot(self.lamba, self.expected_features(self.lamba)) - return llh1 + llh2 - kl, llh1, llh2, kl - # FIXME: KL comes out negative... - - def logz(self, lamba): - lz = 0 - for pid, cid, cnt in edges: - p1 = self.pcm.prob(pid, cid) - q1 = p1 / sum(p1) * exp(-lamba) - q1z = sum(q1) - - p2 = self.cpm.prob(pid, cid) - q2 = p2 / sum(p2) * exp(-lamba) - q2z = sum(q2) - - lz += log(0.5 * (q1z + q2z)) * cnt - return lz - - # z = 1/2 * (sum_y p1(y|x) exp (-lambda . phi(y)) + sum_y p2(y|x) exp (-lambda . phi(y))) - # = 1/2 (z1 + z2) - # d (log z) / dlambda = 1/2 (E_q1 [ -phi ] + E_q2 [ -phi ] ) - def expected_features(self, lamba): - fs = zeros(num_tags) - for pid, cid, cnt in edges: - p1 = self.pcm.prob(pid, cid) - q1 = (p1 / sum(p1)) * exp(-lamba) - fs -= 0.5 * cnt * q1 / sum(q1) - - p2 = self.cpm.prob(pid, cid) - q2 = (p2 / sum(p2)) * exp(-lamba) - fs -= 0.5 * cnt * q2 / sum(q2) - return fs - -if style == 'p2c': - m = PhraseToContextModel() -elif style == 'c2p': - m = ContextToPhraseModel() -elif style == 'prod': - m = ProductModel() -elif style == 'prodslack': - m = RegularisedProductModel(0.5) -elif style == 'sum': - m = InterpolatedModel(0.5) - -for iteration in range(30): - obj = m.expectation_maximisation_step() - print 'iteration', iteration, 'objective', obj - -for pid, cid, cnt in edges: - p = m.prob(pid, cid) - phrase = phrase_type_list[pid] - phrase_str = ' '.join(map(word_type_list.__getitem__, phrase)) - context = context_type_list[cid] - context_str = ' '.join(map(word_type_list.__getitem__, context)) - print '%s\t%s ||| C=%d' % (phrase_str, context_str, argmax(p)) diff --git a/gi/posterior-regularisation/train_pr_global.py b/gi/posterior-regularisation/train_pr_global.py deleted file mode 100644 index 8521bccb..00000000 --- a/gi/posterior-regularisation/train_pr_global.py +++ /dev/null @@ -1,296 +0,0 @@ -import sys -import scipy.optimize -from numpy import * -from numpy.random import random - -# -# Step 1: load the concordance counts -# - -edges_phrase_to_context = [] -edges_context_to_phrase = [] -types = {} -context_types = {} -num_edges = 0 - -for line in sys.stdin: - phrase, rest = line.strip().split('\t') - parts = rest.split('|||') - edges_phrase_to_context.append((phrase, [])) - for i in range(0, len(parts), 2): - context, count = parts[i:i+2] - - ctx = tuple(filter(lambda x: x != '<PHRASE>', context.split())) - cnt = int(count.strip()[2:]) - edges_phrase_to_context[-1][1].append((ctx, cnt)) - - cid = context_types.get(ctx, len(context_types)) - if cid == len(context_types): - context_types[ctx] = cid - edges_context_to_phrase.append((ctx, [])) - edges_context_to_phrase[cid][1].append((phrase, cnt)) - - for token in ctx: - types.setdefault(token, len(types)) - for token in phrase.split(): - types.setdefault(token, len(types)) - - num_edges += 1 - -print 'Read in', num_edges, 'edges and', len(types), 'word types' - -print 'edges_phrase_to_context', edges_phrase_to_context - -# -# Step 2: initialise the model parameters -# - -num_tags = 10 -num_types = len(types) -num_phrases = len(edges_phrase_to_context) -num_contexts = len(edges_context_to_phrase) -delta = int(sys.argv[1]) -gamma = int(sys.argv[2]) - -def normalise(a): - return a / float(sum(a)) - -# Pr(tag | phrase) -tagDist = [normalise(random(num_tags)+1) for p in range(num_phrases)] -#tagDist = [normalise(array(range(1,num_tags+1))) for p in range(num_phrases)] -# Pr(context at pos i = w | tag) indexed by i, tag, word -#contextWordDist = [[normalise(array(range(1,num_types+1))) for t in range(num_tags)] for i in range(4)] -contextWordDist = [[normalise(random(num_types)+1) for t in range(num_tags)] for i in range(4)] -# PR langrange multipliers -lamba = zeros(2 * num_edges * num_tags) -omega_offset = num_edges * num_tags -lamba_index = {} -next = 0 -for phrase, ccs in edges_phrase_to_context: - for context, count in ccs: - lamba_index[phrase,context] = next - next += num_tags -#print lamba_index - -# -# Step 3: expectation maximisation -# - -for iteration in range(20): - tagCounts = [zeros(num_tags) for p in range(num_phrases)] - contextWordCounts = [[zeros(num_types) for t in range(num_tags)] for i in range(4)] - - #print 'tagDist', tagDist - #print 'contextWordCounts[0][0]', contextWordCounts[0][0] - - # Tune lambda - # dual: min log Z(lamba) s.t. lamba >= 0; - # sum_c lamba_pct <= delta; sum_p lamba_pct <= gamma - def dual(ls): - logz = 0 - for p, (phrase, ccs) in enumerate(edges_phrase_to_context): - for context, count in ccs: - conditionals = zeros(num_tags) - for t in range(num_tags): - prob = tagDist[p][t] - for i in range(4): - prob *= contextWordDist[i][t][types[context[i]]] - conditionals[t] = prob - cz = sum(conditionals) - conditionals /= cz - - #print 'dual', phrase, context, count, 'p =', conditionals - - local_z = 0 - for t in range(num_tags): - li = lamba_index[phrase,context] + t - local_z += conditionals[t] * exp(-ls[li] - ls[omega_offset+li]) - logz += log(local_z) * count - - #print 'ls', ls - #print 'lambda', list(ls) - #print 'dual', logz - return logz - - def loglikelihood(): - llh = 0 - for p, (phrase, ccs) in enumerate(edges_phrase_to_context): - for context, count in ccs: - conditionals = zeros(num_tags) - for t in range(num_tags): - prob = tagDist[p][t] - for i in range(4): - prob *= contextWordDist[i][t][types[context[i]]] - conditionals[t] = prob - cz = sum(conditionals) - llh += log(cz) * count - return llh - - def primal(ls): - # FIXME: returns negative values for KL (impossible) - logz = dual(ls) - expectations = -dual_deriv(ls) - kl = -logz - dot(ls, expectations) - llh = loglikelihood() - - pt_l1linf = 0 - for phrase, ccs in edges_phrase_to_context: - for t in range(num_tags): - best = -1e500 - for context, count in ccs: - li = lamba_index[phrase,context] + t - s = expectations[li] - if s > best: best = s - pt_l1linf += best - - ct_l1linf = 0 - for context, pcs in edges_context_to_phrase: - for t in range(num_tags): - best = -1e500 - for phrase, count in pcs: - li = omega_offset + lamba_index[phrase,context] + t - s = expectations[li] - if s > best: best = s - ct_l1linf += best - - return llh, kl, pt_l1linf, ct_l1linf, llh - kl - delta * pt_l1linf - gamma * ct_l1linf - - def dual_deriv(ls): - # d/dl log(z) = E_q[phi] - deriv = zeros(2 * num_edges * num_tags) - for p, (phrase, ccs) in enumerate(edges_phrase_to_context): - for context, count in ccs: - conditionals = zeros(num_tags) - for t in range(num_tags): - prob = tagDist[p][t] - for i in range(4): - prob *= contextWordDist[i][t][types[context[i]]] - conditionals[t] = prob - cz = sum(conditionals) - conditionals /= cz - - scores = zeros(num_tags) - for t in range(num_tags): - li = lamba_index[phrase,context] + t - scores[t] = conditionals[t] * exp(-ls[li] - ls[omega_offset + li]) - local_z = sum(scores) - - #print 'ddual', phrase, context, count, 'q =', scores / local_z - - for t in range(num_tags): - deriv[lamba_index[phrase,context] + t] -= count * scores[t] / local_z - deriv[omega_offset + lamba_index[phrase,context] + t] -= count * scores[t] / local_z - - #print 'ddual', list(deriv) - return deriv - - def constraints(ls): - cons = zeros(num_phrases * num_tags + num_edges * num_tags) - - index = 0 - for phrase, ccs in edges_phrase_to_context: - for t in range(num_tags): - if delta > 0: - total = delta - for cprime, count in ccs: - total -= ls[lamba_index[phrase, cprime] + t] - cons[index] = total - index += 1 - - for context, pcs in edges_context_to_phrase: - for t in range(num_tags): - if gamma > 0: - total = gamma - for pprime, count in pcs: - total -= ls[omega_offset + lamba_index[pprime, context] + t] - cons[index] = total - index += 1 - - #print 'cons', cons - return cons - - def constraints_deriv(ls): - cons = zeros((num_phrases * num_tags + num_edges * num_tags, 2 * num_edges * num_tags)) - - index = 0 - for phrase, ccs in edges_phrase_to_context: - for t in range(num_tags): - if delta > 0: - d = cons[index,:]#zeros(num_edges * num_tags) - for cprime, count in ccs: - d[lamba_index[phrase, cprime] + t] = -1 - #cons[index] = d - index += 1 - - for context, pcs in edges_context_to_phrase: - for t in range(num_tags): - if gamma > 0: - d = cons[index,:]#d = zeros(num_edges * num_tags) - for pprime, count in pcs: - d[omega_offset + lamba_index[pprime, context] + t] = -1 - #cons[index] = d - index += 1 - #print 'dcons', cons - return cons - - print 'Pre lambda optimisation dual', dual(lamba), 'primal', primal(lamba) - #print 'lambda', lamba, lamba.shape - #print 'bounds', [(0, max(delta, gamma))] * (2 * num_edges * num_tags) - - lamba = scipy.optimize.fmin_slsqp(dual, lamba, - bounds=[(0, max(delta, gamma))] * (2 * num_edges * num_tags), - f_ieqcons=constraints, - fprime=dual_deriv, - fprime_ieqcons=constraints_deriv, - iprint=0) - print 'Post lambda optimisation dual', dual(lamba), 'primal', primal(lamba) - - # E-step - llh = log_z = 0 - for p, (phrase, ccs) in enumerate(edges_phrase_to_context): - for context, count in ccs: - conditionals = zeros(num_tags) - for t in range(num_tags): - prob = tagDist[p][t] - for i in range(4): - prob *= contextWordDist[i][t][types[context[i]]] - conditionals[t] = prob - cz = sum(conditionals) - conditionals /= cz - llh += log(cz) * count - - q = zeros(num_tags) - li = lamba_index[phrase, context] - for t in range(num_tags): - q[t] = conditionals[t] * exp(-lamba[li + t] - lamba[omega_offset + li + t]) - qz = sum(q) - log_z += count * log(qz) - - for t in range(num_tags): - tagCounts[p][t] += count * q[t] / qz - - for i in range(4): - for t in range(num_tags): - contextWordCounts[i][t][types[context[i]]] += count * q[t] / qz - - print 'iteration', iteration, 'llh', llh, 'logz', log_z - - # M-step - for p in range(num_phrases): - tagDist[p] = normalise(tagCounts[p]) - for i in range(4): - for t in range(num_tags): - contextWordDist[i][t] = normalise(contextWordCounts[i][t]) - -for p, (phrase, ccs) in enumerate(edges_phrase_to_context): - for context, count in ccs: - conditionals = zeros(num_tags) - for t in range(num_tags): - prob = tagDist[p][t] - for i in range(4): - prob *= contextWordDist[i][t][types[context[i]]] - conditionals[t] = prob - cz = sum(conditionals) - conditionals /= cz - - print '%s\t%s ||| C=%d |||' % (phrase, context, argmax(conditionals)), conditionals diff --git a/gi/posterior-regularisation/train_pr_parallel.py b/gi/posterior-regularisation/train_pr_parallel.py deleted file mode 100644 index 3b9cefed..00000000 --- a/gi/posterior-regularisation/train_pr_parallel.py +++ /dev/null @@ -1,333 +0,0 @@ -import sys -import scipy.optimize -from numpy import * -from numpy.random import random, seed - -# -# Step 1: load the concordance counts -# - -edges_phrase_to_context = [] -edges_context_to_phrase = [] -types = {} -context_types = {} -num_edges = 0 - -for line in sys.stdin: - phrase, rest = line.strip().split('\t') - parts = rest.split('|||') - edges_phrase_to_context.append((phrase, [])) - for i in range(0, len(parts), 2): - context, count = parts[i:i+2] - - ctx = tuple(filter(lambda x: x != '<PHRASE>', context.split())) - cnt = int(count.strip()[2:]) - edges_phrase_to_context[-1][1].append((ctx, cnt)) - - cid = context_types.get(ctx, len(context_types)) - if cid == len(context_types): - context_types[ctx] = cid - edges_context_to_phrase.append((ctx, [])) - edges_context_to_phrase[cid][1].append((phrase, cnt)) - - for token in ctx: - types.setdefault(token, len(types)) - for token in phrase.split(): - types.setdefault(token, len(types)) - - num_edges += 1 - -# -# Step 2: initialise the model parameters -# - -num_tags = 25 -num_types = len(types) -num_phrases = len(edges_phrase_to_context) -num_contexts = len(edges_context_to_phrase) -delta = float(sys.argv[1]) -assert sys.argv[2] in ('local', 'global') -local = sys.argv[2] == 'local' -if len(sys.argv) >= 2: - seed(int(sys.argv[3])) - -print 'Read in', num_edges, 'edges', num_phrases, 'phrases', num_contexts, 'contexts and', len(types), 'word types' - -def normalise(a): - return a / float(sum(a)) - -# Pr(tag | phrase) -tagDist = [normalise(random(num_tags)+1) for p in range(num_phrases)] -# Pr(context at pos i = w | tag) indexed by i, tag, word -contextWordDist = [[normalise(random(num_types)+1) for t in range(num_tags)] for i in range(4)] - -# -# Step 3: expectation maximisation -# - -class GlobalDualObjective: - """ - Objective, log(z), for all phrases s.t. lambda >= 0, sum_c lambda_pct <= scale - """ - - def __init__(self, scale): - self.scale = scale - self.posterior = zeros((num_edges, num_tags)) - self.q = zeros((num_edges, num_tags)) - self.llh = 0 - - index = 0 - for j, (phrase, edges) in enumerate(edges_phrase_to_context): - for context, count in edges: - for t in range(num_tags): - prob = tagDist[j][t] - for k, token in enumerate(context): - prob *= contextWordDist[k][t][types[token]] - self.posterior[index,t] = prob - z = sum(self.posterior[index,:]) - self.posterior[index,:] /= z - self.llh += log(z) * count - index += 1 - - def objective(self, ls): - ls = ls.reshape((num_edges, num_tags)) - logz = 0 - - index = 0 - for j, (phrase, edges) in enumerate(edges_phrase_to_context): - for context, count in edges: - for t in range(num_tags): - self.q[index,t] = self.posterior[index,t] * exp(-ls[index,t]) - local_z = sum(self.q[index,:]) - self.q[index,:] /= local_z - logz += log(local_z) * count - index += 1 - - return logz - - # FIXME: recomputes q many more times than necessary - - def gradient(self, ls): - ls = ls.reshape((num_edges, num_tags)) - gradient = zeros((num_edges, num_tags)) - - index = 0 - for j, (phrase, edges) in enumerate(edges_phrase_to_context): - for context, count in edges: - for t in range(num_tags): - self.q[index,t] = self.posterior[index,t] * exp(-ls[index,t]) - local_z = sum(self.q[index,:]) - self.q[index,:] /= local_z - for t in range(num_tags): - gradient[index,t] -= self.q[index,t] * count - index += 1 - - return gradient.ravel() - - def constraints(self, ls): - ls = ls.reshape((num_edges, num_tags)) - cons = ones((num_phrases, num_tags)) * self.scale - index = 0 - for j, (phrase, edges) in enumerate(edges_phrase_to_context): - for i, (context, count) in enumerate(edges): - for t in range(num_tags): - cons[j,t] -= ls[index,t] * count - index += 1 - return cons.ravel() - - def constraints_gradient(self, ls): - ls = ls.reshape((num_edges, num_tags)) - gradient = zeros((num_phrases, num_tags, num_edges, num_tags)) - index = 0 - for j, (phrase, edges) in enumerate(edges_phrase_to_context): - for i, (context, count) in enumerate(edges): - for t in range(num_tags): - gradient[j,t,index,t] -= count - index += 1 - return gradient.reshape((num_phrases*num_tags, num_edges*num_tags)) - - def optimize(self): - ls = zeros(num_edges * num_tags) - #print '\tpre lambda optimisation dual', self.objective(ls) #, 'primal', primal(lamba) - ls = scipy.optimize.fmin_slsqp(self.objective, ls, - bounds=[(0, self.scale)] * num_edges * num_tags, - f_ieqcons=self.constraints, - fprime=self.gradient, - fprime_ieqcons=self.constraints_gradient, - iprint=0) # =2 for verbose - #print '\tpost lambda optimisation dual', self.objective(ls) #, 'primal', primal(lamba) - - # returns llh, kl and l1lmax contribution - l1lmax = 0 - index = 0 - for j, (phrase, edges) in enumerate(edges_phrase_to_context): - for t in range(num_tags): - lmax = None - for i, (context, count) in enumerate(edges): - lmax = max(lmax, self.q[index+i,t]) - l1lmax += lmax - index += len(edges) - - return self.llh, -self.objective(ls) + dot(ls, self.gradient(ls)), l1lmax - -class LocalDualObjective: - """ - Local part of objective, log(z) relevant to lambda_p**. - Optimised subject to lambda >= 0, sum_c lambda_pct <= scale forall t - """ - - def __init__(self, phraseId, scale): - self.phraseId = phraseId - self.scale = scale - edges = edges_phrase_to_context[self.phraseId][1] - self.posterior = zeros((len(edges), num_tags)) - self.q = zeros((len(edges), num_tags)) - self.llh = 0 - - for i, (context, count) in enumerate(edges): - for t in range(num_tags): - prob = tagDist[phraseId][t] - for j, token in enumerate(context): - prob *= contextWordDist[j][t][types[token]] - self.posterior[i,t] = prob - z = sum(self.posterior[i,:]) - self.posterior[i,:] /= z - self.llh += log(z) * count - - def objective(self, ls): - edges = edges_phrase_to_context[self.phraseId][1] - ls = ls.reshape((len(edges), num_tags)) - logz = 0 - - for i, (context, count) in enumerate(edges): - for t in range(num_tags): - self.q[i,t] = self.posterior[i,t] * exp(-ls[i,t]) - local_z = sum(self.q[i,:]) - self.q[i,:] /= local_z - logz += log(local_z) * count - - return logz - - # FIXME: recomputes q many more times than necessary - - def gradient(self, ls): - edges = edges_phrase_to_context[self.phraseId][1] - ls = ls.reshape((len(edges), num_tags)) - gradient = zeros((len(edges), num_tags)) - - for i, (context, count) in enumerate(edges): - for t in range(num_tags): - self.q[i,t] = self.posterior[i,t] * exp(-ls[i,t]) - local_z = sum(self.q[i,:]) - self.q[i,:] /= local_z - for t in range(num_tags): - gradient[i,t] -= self.q[i,t] * count - - return gradient.ravel() - - def constraints(self, ls): - edges = edges_phrase_to_context[self.phraseId][1] - ls = ls.reshape((len(edges), num_tags)) - cons = ones(num_tags) * self.scale - for t in range(num_tags): - for i, (context, count) in enumerate(edges): - cons[t] -= ls[i,t] * count - return cons - - def constraints_gradient(self, ls): - edges = edges_phrase_to_context[self.phraseId][1] - ls = ls.reshape((len(edges), num_tags)) - gradient = zeros((num_tags, len(edges), num_tags)) - for t in range(num_tags): - for i, (context, count) in enumerate(edges): - gradient[t,i,t] -= count - return gradient.reshape((num_tags, len(edges)*num_tags)) - - def optimize(self, ls=None): - edges = edges_phrase_to_context[self.phraseId][1] - if ls == None: - ls = zeros(len(edges) * num_tags) - #print '\tpre lambda optimisation dual', self.objective(ls) #, 'primal', primal(lamba) - ls = scipy.optimize.fmin_slsqp(self.objective, ls, - bounds=[(0, self.scale)] * len(edges) * num_tags, - f_ieqcons=self.constraints, - fprime=self.gradient, - fprime_ieqcons=self.constraints_gradient, - iprint=0) # =2 for verbose - #print '\tlambda', list(ls) - #print '\tpost lambda optimisation dual', self.objective(ls) #, 'primal', primal(lamba) - - # returns llh, kl and l1lmax contribution - l1lmax = 0 - for t in range(num_tags): - lmax = None - for i, (context, count) in enumerate(edges): - lmax = max(lmax, self.q[i,t]) - l1lmax += lmax - - return self.llh, -self.objective(ls) + dot(ls, self.gradient(ls)), l1lmax, ls - -ls = [None] * num_phrases -for iteration in range(20): - tagCounts = [zeros(num_tags) for p in range(num_phrases)] - contextWordCounts = [[zeros(num_types) for t in range(num_tags)] for i in range(4)] - - # E-step - llh = kl = l1lmax = 0 - if local: - for p in range(num_phrases): - o = LocalDualObjective(p, delta) - #print '\toptimising lambda for phrase', p, '=', edges_phrase_to_context[p][0] - #print '\toptimising lambda for phrase', p, 'ls', ls[p] - obj = o.optimize(ls[p]) - #print '\tphrase', p, 'deltas', obj - llh += obj[0] - kl += obj[1] - l1lmax += obj[2] - ls[p] = obj[3] - - edges = edges_phrase_to_context[p][1] - for j, (context, count) in enumerate(edges): - for t in range(num_tags): - tagCounts[p][t] += count * o.q[j,t] - for i in range(4): - for t in range(num_tags): - contextWordCounts[i][t][types[context[i]]] += count * o.q[j,t] - - #print 'iteration', iteration, 'LOCAL objective', (llh + kl + delta * l1lmax), 'llh', llh, 'kl', kl, 'l1lmax', l1lmax - else: - o = GlobalDualObjective(delta) - obj = o.optimize() - llh, kl, l1lmax = o.optimize() - - index = 0 - for p, (phrase, edges) in enumerate(edges_phrase_to_context): - for context, count in edges: - for t in range(num_tags): - tagCounts[p][t] += count * o.q[index,t] - for i in range(4): - for t in range(num_tags): - contextWordCounts[i][t][types[context[i]]] += count * o.q[index,t] - index += 1 - - print 'iteration', iteration, 'objective', (llh - kl - delta * l1lmax), 'llh', llh, 'kl', kl, 'l1lmax', l1lmax - - # M-step - for p in range(num_phrases): - tagDist[p] = normalise(tagCounts[p]) - for i in range(4): - for t in range(num_tags): - contextWordDist[i][t] = normalise(contextWordCounts[i][t]) - -for p, (phrase, ccs) in enumerate(edges_phrase_to_context): - for context, count in ccs: - conditionals = zeros(num_tags) - for t in range(num_tags): - prob = tagDist[p][t] - for i in range(4): - prob *= contextWordDist[i][t][types[context[i]]] - conditionals[t] = prob - cz = sum(conditionals) - conditionals /= cz - - print '%s\t%s ||| C=%d |||' % (phrase, context, argmax(conditionals)), conditionals diff --git a/gi/pyp-topics/scripts/contexts2documents.py b/gi/pyp-topics/scripts/contexts2documents.py deleted file mode 100755 index 9be4ebbb..00000000 --- a/gi/pyp-topics/scripts/contexts2documents.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/python - -import sys -from operator import itemgetter - -if len(sys.argv) > 3: - print "Usage: contexts2documents.py [contexts_index_out] [phrases_index_out]" - exit(1) - -context_index = {} -phrase_index = {} -for line in sys.stdin: - phrase, line_tail = line.split('\t') - - raw_contexts = line_tail.split('|||') - contexts = [c.strip() for x,c in enumerate(raw_contexts) if x%2 == 0] - counts = [int(c.split('=')[1].strip()) for x,c in enumerate(raw_contexts) if x%2 != 0] - phrase_index.setdefault(phrase, len(phrase_index)) - print len(contexts), - for context,count in zip(contexts,counts): - c = context_index.setdefault(context, len(context_index)) - print "%d:%d" % (c,count), - print -if 1 < len(sys.argv) < 4: - contexts_out = open(sys.argv[1],'w') - contexts = context_index.items() - contexts.sort(key = itemgetter(1)) - for context in contexts: - print >>contexts_out, context[0] - contexts_out.close() -if len(sys.argv) == 3: - phrases_out = open(sys.argv[2],'w') - phrases = phrase_index.items() - phrases.sort(key = itemgetter(1)) - for phrase in phrases: - print >>phrases_out, phrase[0] - phrases_out.close() diff --git a/gi/pyp-topics/scripts/extract_contexts.py b/gi/pyp-topics/scripts/extract_contexts.py deleted file mode 100755 index b2723f2a..00000000 --- a/gi/pyp-topics/scripts/extract_contexts.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python - -import sys,collections - -def extract_backoff(context_list, order): - assert len(context_list) == (2*order) - backoffs = [] - for i in range(1,order+1): - if i == order: - backoffs.append(([context_list[i-1]+"|"], ["|"+context_list[i]])) - else: - right_limit = 2*order-i - core = context_list[i:right_limit] - left = [context_list[i-1]+"|"*(order-i+1)] - right = ["|"*(order-i+1)+context_list[right_limit]] - backoffs.append((core, left, right)) -# print context_list, backoffs - return backoffs - -def tuple_to_str(t): - s="" - for i,x in enumerate(t): - if i > 0: s += "|" - s += str(x) - return s - -if len(sys.argv) < 3: - print "Usage: extract-contexts.py output_filename order cutoff lowercase" - exit(1) - -output_filename = sys.argv[1] -order = int(sys.argv[2]) -cutoff = 0 -if len(sys.argv) > 3: - cutoff = int(sys.argv[3]) -lowercase = False -if len(sys.argv) > 4: - lowercase = bool(sys.argv[4]) - -contexts_dict={} -contexts_list=[] -contexts_freq=collections.defaultdict(int) -contexts_backoff={} - -token_dict={} -token_list=[] -documents_dict=collections.defaultdict(dict) - -contexts_at_order = [i for i in range(order+1)] - -prefix = ["<s%d>|<s>"%i for i in range(order)] -suffix = ["</s%d>|</s>"%i for i in range(order)] - -for line in sys.stdin: - tokens = list(prefix) - tokens.extend(line.split()) - tokens.extend(suffix) - if lowercase: - tokens = map(lambda x: x.lower(), tokens) - - for i in range(order, len(tokens)-order): - context_list = [] - term="" - for j in range(i-order, i+order+1): - token,tag = tokens[j].rsplit('|',2) - if j != i: - context_list.append(token) - else: - if token not in token_dict: - token_dict[token] = len(token_dict) - token_list.append(token) - term = token_dict[token] - - context = tuple_to_str(tuple(context_list)) - - if context not in contexts_dict: - context_index = len(contexts_dict) - contexts_dict[context] = context_index - contexts_list.append(context) - contexts_at_order[0] += 1 - - # handle backoff - backoff_contexts = extract_backoff(context_list, order) - bo_indexes=[(context_index,)] -# bo_indexes=[(context,)] - for i,bo in enumerate(backoff_contexts): - factor_indexes=[] - for factor in bo: - bo_tuple = tuple_to_str(tuple(factor)) - if bo_tuple not in contexts_dict: - contexts_dict[bo_tuple] = len(contexts_dict) - contexts_list.append(bo_tuple) - contexts_at_order[i+1] += 1 -# factor_indexes.append(bo_tuple) - factor_indexes.append(contexts_dict[bo_tuple]) - bo_indexes.append(tuple(factor_indexes)) - - for i in range(len(bo_indexes)-1): - contexts_backoff[bo_indexes[i][0]] = bo_indexes[i+1] - - context_index = contexts_dict[context] - contexts_freq[context_index] += 1 - - if context_index not in documents_dict[term]: - documents_dict[term][context_index] = 1 - else: - documents_dict[term][context_index] += 1 - -term_file = open(output_filename+".terms",'w') -for t in token_list: print >>term_file, t -term_file.close() - -contexts_file = open(output_filename+".contexts",'w') -for c in contexts_list: - print >>contexts_file, c -contexts_file.close() - -data_file = open(output_filename+".data",'w') -for t in range(len(token_list)): - line="" - num_active=0 - for c in documents_dict[t]: - count = documents_dict[t][c] - if contexts_freq[c] >= cutoff: - line += (' ' + str(c) + ':' + str(count)) - num_active += 1 - if num_active > 0: - print >>data_file, "%d%s" % (num_active,line) -data_file.close() - -contexts_backoff_file = open(output_filename+".contexts_backoff",'w') -print >>contexts_backoff_file, len(contexts_list), order, -#for x in contexts_at_order: -# print >>contexts_backoff_file, x, -#print >>contexts_backoff_file -for x in range(order-1): - print >>contexts_backoff_file, 3, -print >>contexts_backoff_file, 2 - -for x in contexts_backoff: - print >>contexts_backoff_file, x, - for y in contexts_backoff[x]: print >>contexts_backoff_file, y, - print >>contexts_backoff_file -contexts_backoff_file.close() diff --git a/gi/pyp-topics/scripts/extract_contexts_test.py b/gi/pyp-topics/scripts/extract_contexts_test.py deleted file mode 100755 index 693b6e0b..00000000 --- a/gi/pyp-topics/scripts/extract_contexts_test.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/python - -import sys,collections - -def tuple_to_str(t): - s="" - for i,x in enumerate(t): - if i > 0: s += "|" - s += str(x) - return s - -if len(sys.argv) < 5: - print "Usage: extract-contexts_test.py output_filename vocab contexts order lowercase" - exit(1) - -output_filename = sys.argv[1] -output = open(output_filename+".test_data",'w') - -unk_term="-UNK-" -vocab_dict={} -for i,x in enumerate(file(sys.argv[2], 'r').readlines()): - vocab_dict[x.strip()]=i - -contexts_dict={} -contexts_list=[] -for i,x in enumerate(file(sys.argv[3], 'r').readlines()): - contexts_dict[x.strip()]=i - contexts_list.append(x.strip()) - -order = int(sys.argv[4]) - -lowercase = False -if len(sys.argv) > 5: - lowercase = bool(sys.argv[5]) -if lowercase: unk_term = unk_term.lower() - -prefix = ["<s%d>|<s>"%i for i in range(order)] -suffix = ["</s%d>|</s>"%i for i in range(order)] - -assert unk_term in vocab_dict -for line in sys.stdin: - tokens = list(prefix) - tokens.extend(line.split()) - tokens.extend(suffix) - if lowercase: - tokens = map(lambda x: x.lower(), tokens) - - for i in range(order, len(tokens)-order): - context_list=[] - term="" - for j in range(i-order, i+order+1): - token,tag = tokens[j].rsplit('|',2) - if j != i: - context_list.append(token) - else: - if token not in vocab_dict: - term = vocab_dict[unk_term] - else: - term = vocab_dict[token] - context = tuple_to_str(context_list) - if context not in contexts_dict: - contexts_dict[context] = len(contexts_dict) - contexts_list.append(context) - context_index = contexts_dict[context] - print >>output, "%d:%d" % (term,context_index), - print >>output -output.close() - -contexts_file = open(output_filename+".test_contexts",'w') -for c in contexts_list: - print >>contexts_file, c -contexts_file.close() diff --git a/gi/pyp-topics/scripts/extract_leaves.py b/gi/pyp-topics/scripts/extract_leaves.py deleted file mode 100755 index 14783b36..00000000 --- a/gi/pyp-topics/scripts/extract_leaves.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/python - -import nltk -import nltk.probability -import sys -import getopt - -lexicalise=False -rm_traces=False -cutoff=100 -length_cutoff=10000 -try: - opts, args = getopt.getopt(sys.argv[1:], "hs:c:l", ["help", "lexicalise", "cutoff","sentence-length","remove-traces"]) -except getopt.GetoptError: - print "Usage: extract_leaves.py [-lsc]" - sys.exit(2) -for opt, arg in opts: - if opt in ("-h", "--help"): - print "Usage: extract_leaves.py [-lsc]" - sys.exit() - elif opt in ("-l", "--lexicalise"): - lexicalise = True - elif opt in ("-c", "--cutoff"): - cutoff = int(arg) - elif opt in ("-s", "--sentence-length"): - length_cutoff = int(arg) - elif opt in ("--remove-traces"): - rm_traces = True - -token_freq = nltk.probability.FreqDist() -lines = [] -for line in sys.stdin: - t = nltk.Tree.parse(line) - pos = t.pos() - if len(pos) <= length_cutoff: - lines.append(pos) - for token, tag in pos: - token_freq.inc(token) - -for line in lines: - for token,tag in line: - if not (rm_traces and tag == "-NONE-"): - if lexicalise: - if token_freq[token] < cutoff: - token = '-UNK-' - print '%s|%s' % (token,tag), - else: - print '%s' % tag, - print diff --git a/gi/pyp-topics/scripts/map-documents.py b/gi/pyp-topics/scripts/map-documents.py deleted file mode 100755 index 703de312..00000000 --- a/gi/pyp-topics/scripts/map-documents.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/python - -import sys - -if len(sys.argv) != 2: - print "Usage: map-documents.py vocab-file" - exit(1) - -vocab = file(sys.argv[1], 'r').readlines() -term_dict = map(lambda x: x.strip(), vocab) - -for line in sys.stdin: - tokens = line.split() - for token in tokens: - elements = token.split(':') - if len(elements) == 1: - print "%s" % (term_dict[int(elements[0])]), - else: - print "%s:%s" % (term_dict[int(elements[0])], elements[1]), - print diff --git a/gi/pyp-topics/scripts/map-terms.py b/gi/pyp-topics/scripts/map-terms.py deleted file mode 100755 index eb0298d7..00000000 --- a/gi/pyp-topics/scripts/map-terms.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/python - -import sys - -if len(sys.argv) != 2: - print "Usage: map-terms.py vocab-file" - exit(1) - -vocab = file(sys.argv[1], 'r').readlines() -term_dict = map(lambda x: x.strip().replace(' ','_'), vocab) - -for line in sys.stdin: - tokens = line.split() - for token in tokens: - elements = token.split(':') - if len(elements) == 1: - print "%s" % (term_dict[int(elements[0])]), - else: - print "%s:%s" % (term_dict[int(elements[0])], elements[1]), - print diff --git a/gi/pyp-topics/scripts/run.sh b/gi/pyp-topics/scripts/run.sh deleted file mode 100644 index 19e625b1..00000000 --- a/gi/pyp-topics/scripts/run.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - - -./simple-extract-context.sh ~/workspace/clsp2010/jhuws2010/data/btec/split.zh-en.al 1 | ~/workspace/pyp-topics/scripts/contexts2documents.py > split.zh-en.data - -~/workspace/pyp-topics/bin/pyp-topics-train -d split.zh-en.data -t 50 -s 100 -o split.zh-en.documents.gz -w split.zh-en.topics.gz -gunzip split.zh-en.documents.gz - -~/workspace/cdec/extools/extractor -i ../jhuws2010/data/btec/split.zh-en.al -S 1 -c 500000 -L 12 --base_phrase_spans | ~/workspace/pyp-topics/scripts/spans2labels.py split.zh-en.phrases split.zh-en.contexts split.zh-en.documents > corpus.zh-en.labelled_spans - -paste -d " " ~/workspace/clsp2010/jhuws2010/data/btec/split.zh-en.al corpus.labelled_spans > split.zh-en.labelled_spans - -./simple-extract.sh ~/workspace/clsp2010/scratch/split.zh-en.labelled_spans diff --git a/gi/pyp-topics/scripts/score-mkcls.py b/gi/pyp-topics/scripts/score-mkcls.py deleted file mode 100755 index 6bd33fc5..00000000 --- a/gi/pyp-topics/scripts/score-mkcls.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python - -import sys -from collections import defaultdict - -def dict_max(d): - max_val=-1 - max_key=None - for k in d: - if d[k] > max_val: - max_val = d[k] - max_key = k - assert max_key - return max_key - -if len(sys.argv) != 3: - print "Usage: score-mkcls.py gold classes" - exit(1) - -gold_file=open(sys.argv[1],'r') - -term_to_topics = {} -for line in open(sys.argv[2],'r'): - term,cls = line.split() - term_to_topics[term] = cls - -gold_to_topics = defaultdict(dict) -topics_to_gold = defaultdict(dict) - -for gold_line in gold_file: - gold_tokens = gold_line.split() - for gold_token in gold_tokens: - gold_term,gold_tag = gold_token.rsplit('|',1) - pred_token = term_to_topics[gold_term] - gold_to_topics[gold_tag][pred_token] \ - = gold_to_topics[gold_tag].get(pred_token, 0) + 1 - topics_to_gold[pred_token][gold_tag] \ - = topics_to_gold[pred_token].get(gold_tag, 0) + 1 - -pred=0 -correct=0 -gold_file=open(sys.argv[1],'r') -for gold_line in gold_file: - gold_tokens = gold_line.split() - - for gold_token in gold_tokens: - gold_term,gold_tag = gold_token.rsplit('|',1) - pred_token = term_to_topics[gold_term] - print "%s|%s|%s" % (gold_token, pred_token, dict_max(topics_to_gold[pred_token])), - pred += 1 - if gold_tag == dict_max(topics_to_gold[pred_token]): - correct += 1 - print -print >>sys.stderr, "Many-to-One Accuracy = %f" % (float(correct) / pred) -#for x in gold_to_topics: -# print x,dict_max(gold_to_topics[x]) -#print "###################################################" -#for x in range(len(topics_to_gold)): -# print x,dict_max(topics_to_gold[str(x)]) -# print x,topics_to_gold[str(x)] -#print term_to_topics diff --git a/gi/pyp-topics/scripts/score-topics.py b/gi/pyp-topics/scripts/score-topics.py deleted file mode 100755 index 1d8a1fcd..00000000 --- a/gi/pyp-topics/scripts/score-topics.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/python - -import sys -from collections import defaultdict - -def dict_max(d): - max_val=-1 - max_key=None - for k in d: - if d[k] > max_val: - max_val = d[k] - max_key = k - assert max_key - return max_key - -if len(sys.argv) != 3: - print "Usage: score-topics.py gold pred" - exit(1) - -gold_file=open(sys.argv[1],'r') -pred_file=open(sys.argv[2],'r') - -gold_to_topics = defaultdict(dict) -topics_to_gold = defaultdict(dict) -term_to_topics = defaultdict(dict) - -for gold_line,pred_line in zip(gold_file,pred_file): - gold_tokens = gold_line.split() - pred_tokens = pred_line.split() - assert len(gold_tokens) == len(pred_tokens) - - for gold_token,pred_token in zip(gold_tokens,pred_tokens): - gold_term,gold_tag = gold_token.rsplit('|',1) - gold_to_topics[gold_tag][pred_token] \ - = gold_to_topics[gold_tag].get(pred_token, 0) + 1 - term_to_topics[gold_term][pred_token] \ - = term_to_topics[gold_term].get(pred_token, 0) + 1 - topics_to_gold[pred_token][gold_tag] \ - = topics_to_gold[pred_token].get(gold_tag, 0) + 1 - -pred=0 -correct=0 -gold_file=open(sys.argv[1],'r') -pred_file=open(sys.argv[2],'r') -for gold_line,pred_line in zip(gold_file,pred_file): - gold_tokens = gold_line.split() - pred_tokens = pred_line.split() - - for gold_token,pred_token in zip(gold_tokens,pred_tokens): - gold_term,gold_tag = gold_token.rsplit('|',1) -# print "%s|%s" % (gold_token, dict_max(gold_to_topics[gold_tag])), - print "%s|%s|%s" % (gold_token, pred_token, dict_max(topics_to_gold[pred_token])), - pred += 1 - if gold_tag == dict_max(topics_to_gold[pred_token]): - correct += 1 - print -print >>sys.stderr, "Many-to-One Accuracy = %f" % (float(correct) / pred) -#for x in gold_to_topics: -# print x,dict_max(gold_to_topics[x]) -#print "###################################################" -#for x in range(len(topics_to_gold)): -# print x,dict_max(topics_to_gold[str(x)]) -# print x,topics_to_gold[str(x)] -#print term_to_topics diff --git a/gi/pyp-topics/scripts/spans2labels.py b/gi/pyp-topics/scripts/spans2labels.py deleted file mode 100755 index 50fa8106..00000000 --- a/gi/pyp-topics/scripts/spans2labels.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/python - -import sys -from operator import itemgetter - -if len(sys.argv) <= 2: - print "Usage: spans2labels.py phrase_context_index [order] [threshold] [languages={s,t,b}{s,t,b}] [type={tag,tok,both},{tag,tok,both}]" - exit(1) - -order=1 -threshold = 0 -cutoff_cat = "<UNK>" -if len(sys.argv) > 2: - order = int(sys.argv[2]) -if len(sys.argv) > 3: - threshold = float(sys.argv[3]) -phr=ctx='t' -if len(sys.argv) > 4: - phr, ctx = sys.argv[4] - assert phr in 'stb' - assert ctx in 'stb' -phr_typ = ctx_typ = 'both' -if len(sys.argv) > 5: - phr_typ, ctx_typ = sys.argv[5].split(',') - assert phr_typ in ('tag', 'tok', 'both') - assert ctx_typ in ('tag', 'tok', 'both') - -#print >>sys.stderr, "Loading phrase index" -phrase_context_index = {} -for line in file(sys.argv[1], 'r'): - phrase,tail= line.split('\t') - contexts = tail.split(" ||| ") - try: # remove Phil's bizarre integer pair - x,y = contexts[0].split() - x=int(x); y=int(y) - contexts = contexts[1:] - except: - pass - if len(contexts) == 1: continue - assert len(contexts) % 2 == 0 - for i in range(0, len(contexts), 2): - #parse contexts[i+1] = " C=1 P=0.8 ... " - features=dict([ keyval.split('=') for keyval in contexts[i+1].split()]) - category = features['C'] - if features.has_key('P') and float(features['P']) < threshold: - category = cutoff_cat - - phrase_context_index[(phrase,contexts[i])] = category - #print (phrase,contexts[i]), category - -#print >>sys.stderr, "Labelling spans" -for line in sys.stdin: - #print >>sys.stderr, "line", line.strip() - line_segments = line.split(' ||| ') - assert len(line_segments) >= 3 - source = ['<s>' for x in range(order)] + line_segments[0].split() + ['</s>' for x in range(order)] - target = ['<s>' for x in range(order)] + line_segments[1].split() + ['</s>' for x in range(order)] - phrases = [ [int(i) for i in x.split('-')] for x in line_segments[2].split()] - - if phr_typ != 'both' or ctx_typ != 'both': - if phr in 'tb' or ctx in 'tb': - target_toks = ['<s>' for x in range(order)] + map(lambda x: x.rsplit('_', 1)[0], line_segments[1].split()) + ['</s>' for x in range(order)] - target_tags = ['<s>' for x in range(order)] + map(lambda x: x.rsplit('_', 1)[-1], line_segments[1].split()) + ['</s>' for x in range(order)] - - if phr in 'tb': - if phr_typ == 'tok': - targetP = target_toks - elif phr_typ == 'tag': - targetP = target_tags - if ctx in 'tb': - if ctx_typ == 'tok': - targetC = target_toks - elif ctx_typ == 'tag': - targetC = target_tags - - if phr in 'sb' or ctx in 'sb': - source_toks = ['<s>' for x in range(order)] + map(lambda x: x.rsplit('_', 1)[0], line_segments[0].split()) + ['</s>' for x in range(order)] - source_tags = ['<s>' for x in range(order)] + map(lambda x: x.rsplit('_', 1)[-1], line_segments[0].split()) + ['</s>' for x in range(order)] - - if phr in 'sb': - if phr_typ == 'tok': - sourceP = source_toks - elif phr_typ == 'tag': - sourceP = source_tags - if ctx in 'sb': - if ctx_typ == 'tok': - sourceC = source_toks - elif ctx_typ == 'tag': - sourceC = source_tags - else: - sourceP = sourceC = source - targetP = targetC = target - - #print >>sys.stderr, "line", source, '---', target, 'phrases', phrases - - print "|||", - - for s1,s2,t1,t2 in phrases: - s1 += order - s2 += order - t1 += order - t2 += order - - phraset = phrases = contextt = contexts = '' - if phr in 'tb': - phraset = reduce(lambda x, y: x+y+" ", targetP[t1:t2], "").strip() - if phr in 'sb': - phrases = reduce(lambda x, y: x+y+" ", sourceP[s1:s2], "").strip() - - if ctx in 'tb': - left_context = reduce(lambda x, y: x+y+" ", targetC[t1-order:t1], "") - right_context = reduce(lambda x, y: x+y+" ", targetC[t2:t2+order], "").strip() - contextt = "%s<PHRASE> %s" % (left_context, right_context) - if ctx in 'sb': - left_context = reduce(lambda x, y: x+y+" ", sourceC[s1-order:s1], "") - right_context = reduce(lambda x, y: x+y+" ", sourceC[s2:s2+order], "").strip() - contexts = "%s<PHRASE> %s" % (left_context, right_context) - - if phr == 'b': - phrase = phraset + ' <SPLIT> ' + phrases - elif phr == 's': - phrase = phrases - else: - phrase = phraset - - if ctx == 'b': - context = contextt + ' <SPLIT> ' + contexts - elif ctx == 's': - context = contexts - else: - context = contextt - - #print "%d-%d-%d-%d looking up" % (s1-order,s2-order,t1-order,t2-order), (phrase, context) - label = phrase_context_index.get((phrase,context), cutoff_cat) - if label != cutoff_cat: #cutoff'd spans are left unlabelled - print "%d-%d-%d-%d:X%s" % (s1-order,s2-order,t1-order,t2-order,label), - print diff --git a/gi/pyp-topics/scripts/tokens2classes.py b/gi/pyp-topics/scripts/tokens2classes.py deleted file mode 100755 index 33df255f..00000000 --- a/gi/pyp-topics/scripts/tokens2classes.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/python - -import sys - -if len(sys.argv) != 3: - print "Usage: tokens2classes.py source_classes target_classes" - exit(1) - -source_to_topics = {} -for line in open(sys.argv[1],'r'): - term,cls = line.split() - source_to_topics[term] = cls - -target_to_topics = {} -for line in open(sys.argv[2],'r'): - term,cls = line.split() - target_to_topics[term] = cls - -for line in sys.stdin: - source, target, tail = line.split(" ||| ") - - for token in source.split(): - print source_to_topics[token], - print "|||", - for token in target.split(): - print target_to_topics[token], - print "|||", tail, diff --git a/gi/pyp-topics/scripts/topics.py b/gi/pyp-topics/scripts/topics.py deleted file mode 100755 index 0db1af71..00000000 --- a/gi/pyp-topics/scripts/topics.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/python - -import sys - -if len(sys.argv) != 2: - print "Usage: topics.py words-per-topic" - exit(1) - -for t,line in enumerate(sys.stdin): - tokens = line.split() - terms = [] - for token in tokens: - elements = token.rsplit(':',1) - terms.append((int(elements[1]),elements[0])) - terms.sort() - terms.reverse() - - print "Topic %d:" % t - map(lambda (x,y) : sys.stdout.write(" %s:%s\n" % (y,x)), terms[:int(sys.argv[1])]) - print diff --git a/gi/pyp-topics/src/Makefile.am b/gi/pyp-topics/src/Makefile.am deleted file mode 100644 index d3f95d0b..00000000 --- a/gi/pyp-topics/src/Makefile.am +++ /dev/null @@ -1,16 +0,0 @@ -bin_PROGRAMS = pyp-topics-train pyp-contexts-train #mpi-pyp-contexts-train - -contexts_lexer.cc: contexts_lexer.l - $(LEX) -s -CF -8 -o$@ $< - -pyp_topics_train_SOURCES = mt19937ar.c corpus.cc gzstream.cc pyp-topics.cc train.cc contexts_lexer.cc contexts_corpus.cc -pyp_topics_train_LDADD = $(top_srcdir)/utils/libutils.a -lz - -pyp_contexts_train_SOURCES = mt19937ar.c corpus.cc gzstream.cc pyp-topics.cc contexts_lexer.cc contexts_corpus.cc train-contexts.cc -pyp_contexts_train_LDADD = $(top_srcdir)/utils/libutils.a -lz - -#mpi_pyp_contexts_train_SOURCES = mt19937ar.c corpus.cc gzstream.cc mpi-pyp-topics.cc contexts_lexer.cc contexts_corpus.cc mpi-train-contexts.cc -#mpi_pyp_contexts_train_LDADD = $(top_srcdir)/utils/libutils.a -lz - -AM_CPPFLAGS = -W -Wall -Wno-sign-compare -funroll-loops -I../../../utils - diff --git a/gi/pyp-topics/src/Makefile.mpi b/gi/pyp-topics/src/Makefile.mpi deleted file mode 100644 index b7b8a290..00000000 --- a/gi/pyp-topics/src/Makefile.mpi +++ /dev/null @@ -1,26 +0,0 @@ -BLD_ARCH=$(shell uname -s) --include macros.${BLD_ARCH} - -local_objs = mt19937ar.o corpus.o gzstream.o mpi-pyp-topics.o contexts_lexer.o contexts_corpus.o mpi-train-contexts.o - -all: mpi-pyp-contexts-train - --include makefile.depend - -#-----------------------# -# Local stuff -#-----------------------# - -mpi-pyp-contexts-train: mpi-train-contexts.o $(local_objs) - $(CXX) -o $@ $^ $(LDFLAGS) - -.PHONY: depend echo -depend: -#$(CXX) -MM $(CXXFLAGS) *.cc *.c | sed 's/^\(.*\.o:\)/obj\/\1/' > makefile.depend - $(CXX) -MM $(CXXFLAGS) *.cc *.c > makefile.depend - -clean: - rm -f *.o - -#clobber: clean -# rm makefile.depend ../bin/${ARCH}/* diff --git a/gi/pyp-topics/src/clock_gettime_stub.c b/gi/pyp-topics/src/clock_gettime_stub.c deleted file mode 100644 index 4883b7c1..00000000 --- a/gi/pyp-topics/src/clock_gettime_stub.c +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c), MM Weiss - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the MM Weiss nor the names of its contributors - * may be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR - * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, - * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * clock_gettime_stub.c - * gcc -Wall -c clock_gettime_stub.c - * posix realtime functions; MacOS user space glue - */ - -/* @comment - * other possible implementation using intel builtin rdtsc - * rdtsc-workaround: http://www.mcs.anl.gov/~kazutomo/rdtsc.html - * - * we could get the ticks by doing this - * - * __asm __volatile("mov %%ebx, %%esi\n\t" - * "cpuid\n\t" - * "xchg %%esi, %%ebx\n\t" - * "rdtsc" - * : "=a" (a), - * "=d" (d) - * ); - - * we could even replace our tricky sched_yield call by assembly code to get a better accurency, - * anyway the following C stub will satisfy 99% of apps using posix clock_gettime call, - * moreover, the setter version (clock_settime) could be easly written using mach primitives: - * http://www.opensource.apple.com/source/xnu/xnu-${VERSION}/osfmk/man/ (clock_[set|get]_time) - * - * hackers don't be crackers, don't you use a flush toilet? - * - * - * @see draft: ./posix-realtime-stub/posix-realtime-stub.c - * - */ - - -#ifdef __APPLE__ - -#pragma weak clock_gettime - -#include <sys/time.h> -#include <sys/resource.h> -#include <mach/mach.h> -#include <mach/clock.h> -#include <mach/mach_time.h> -#include <errno.h> -#include <unistd.h> -#include <sched.h> - -typedef enum { - CLOCK_REALTIME, - CLOCK_MONOTONIC, - CLOCK_PROCESS_CPUTIME_ID, - CLOCK_THREAD_CPUTIME_ID -} clockid_t; - -static mach_timebase_info_data_t __clock_gettime_inf; - -static int clock_gettime(clockid_t clk_id, struct timespec *tp) { - kern_return_t ret; - clock_serv_t clk; - clock_id_t clk_serv_id; - mach_timespec_t tm; - - uint64_t start, end, delta, nano; - - //task_basic_info_data_t tinfo; - //task_thread_times_info_data_t ttinfo; - //mach_msg_type_number_t tflag; - - int retval = -1; - switch (clk_id) { - case CLOCK_REALTIME: - case CLOCK_MONOTONIC: - clk_serv_id = clk_id == CLOCK_REALTIME ? CALENDAR_CLOCK : SYSTEM_CLOCK; - if (KERN_SUCCESS == (ret = host_get_clock_service(mach_host_self(), clk_serv_id, &clk))) { - if (KERN_SUCCESS == (ret = clock_get_time(clk, &tm))) { - tp->tv_sec = tm.tv_sec; - tp->tv_nsec = tm.tv_nsec; - retval = 0; - } - } - if (KERN_SUCCESS != ret) { - errno = EINVAL; - retval = -1; - } - break; - case CLOCK_PROCESS_CPUTIME_ID: - case CLOCK_THREAD_CPUTIME_ID: - start = mach_absolute_time(); - if (clk_id == CLOCK_PROCESS_CPUTIME_ID) { - getpid(); - } else { - sched_yield(); - } - end = mach_absolute_time(); - delta = end - start; - if (0 == __clock_gettime_inf.denom) { - mach_timebase_info(&__clock_gettime_inf); - } - nano = delta * __clock_gettime_inf.numer / __clock_gettime_inf.denom; - tp->tv_sec = nano * 1e-9; - tp->tv_nsec = nano - (tp->tv_sec * 1e9); - retval = 0; - break; - default: - errno = EINVAL; - retval = -1; - } - return retval; -} - -#endif // __APPLE__ - -/* EOF */ diff --git a/gi/pyp-topics/src/contexts_corpus.cc b/gi/pyp-topics/src/contexts_corpus.cc deleted file mode 100644 index 92b1b34c..00000000 --- a/gi/pyp-topics/src/contexts_corpus.cc +++ /dev/null @@ -1,164 +0,0 @@ -#include <sstream> -#include <iostream> -#include <set> - -#include "contexts_corpus.hh" -#include "gzstream.hh" -#include "contexts_lexer.h" - -#include <boost/tuple/tuple.hpp> - - -using namespace std; - -////////////////////////////////////////////////// -// ContextsCorpus -////////////////////////////////////////////////// - -bool read_callback_binary_contexts = false; - -void read_callback(const ContextsLexer::PhraseContextsType& new_contexts, void* extra) { - assert(new_contexts.contexts.size() == new_contexts.counts.size()); - - boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* >* extra_pair - = static_cast< boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* >* >(extra); - - ContextsCorpus* corpus_ptr = extra_pair->get<0>(); - BackoffGenerator* backoff_gen = extra_pair->get<1>(); - //map<string,int>* counts = extra_pair->get<2>(); - - Document* doc(new Document()); - - //cout << "READ: " << new_contexts.phrase << "\t"; - for (int i=0; i < (int)new_contexts.counts.size(); ++i) { - int cache_word_count = corpus_ptr->m_dict.max(); - - //string context_str = corpus_ptr->m_dict.toString(new_contexts.contexts[i]); - int context_index = new_contexts.counts.at(i).first; - string context_str = corpus_ptr->m_dict.toString(new_contexts.contexts[context_index]); - - // filter out singleton contexts - //if (!counts->empty()) { - // map<string,int>::const_iterator find_it = counts->find(context_str); - // if (find_it == counts->end() || find_it->second < 2) - // continue; - //} - - WordID id = corpus_ptr->m_dict.Convert(context_str); - if (cache_word_count != corpus_ptr->m_dict.max()) { - corpus_ptr->m_backoff->terms_at_level(0)++; - corpus_ptr->m_num_types++; - } - - //int count = new_contexts.counts[i]; - int count = new_contexts.counts.at(i).second; - if (read_callback_binary_contexts) { - doc->push_back(id); - corpus_ptr->m_num_terms++; - } - else { - for (int j=0; j<count; ++j) - doc->push_back(id); - corpus_ptr->m_num_terms += count; - } - - // generate the backoff map - if (backoff_gen) { - int order = 1; - WordID backoff_id = id; - //ContextsLexer::Context backedoff_context = new_contexts.contexts[i]; - ContextsLexer::Context backedoff_context = new_contexts.contexts[context_index]; - while (true) { - if (!corpus_ptr->m_backoff->has_backoff(backoff_id)) { - //cerr << "Backing off from " << corpus_ptr->m_dict.Convert(backoff_id) << " to "; - backedoff_context = (*backoff_gen)(backedoff_context); - - if (backedoff_context.empty()) { - //cerr << "Nothing." << endl; - (*corpus_ptr->m_backoff)[backoff_id] = -1; - break; - } - - if (++order > corpus_ptr->m_backoff->order()) - corpus_ptr->m_backoff->order(order); - - int cache_word_count = corpus_ptr->m_dict.max(); - int new_backoff_id = corpus_ptr->m_dict.Convert(backedoff_context); - if (cache_word_count != corpus_ptr->m_dict.max()) - corpus_ptr->m_backoff->terms_at_level(order-1)++; - - //cerr << corpus_ptr->m_dict.Convert(new_backoff_id) << " ." << endl; - - backoff_id = ((*corpus_ptr->m_backoff)[backoff_id] = new_backoff_id); - } - else break; - } - } - //cout << context_str << " (" << id << ") ||| C=" << count << " ||| "; - } - //cout << endl; - - //if (!doc->empty()) { - corpus_ptr->m_documents.push_back(doc); - corpus_ptr->m_keys.push_back(new_contexts.phrase); - //} -} - -void filter_callback(const ContextsLexer::PhraseContextsType& new_contexts, void* extra) { - assert(new_contexts.contexts.size() == new_contexts.counts.size()); - - map<string,int>* context_counts = (static_cast<map<string,int>*>(extra)); - - for (int i=0; i < (int)new_contexts.counts.size(); ++i) { - int context_index = new_contexts.counts.at(i).first; - int count = new_contexts.counts.at(i).second; - //if (read_callback_binary_contexts) count = 1; - //int count = new_contexts.counts[i]; - pair<map<string,int>::iterator,bool> result - = context_counts->insert(make_pair(Dict::toString(new_contexts.contexts[context_index]),count)); - //= context_counts->insert(make_pair(Dict::toString(new_contexts.contexts[i]),count)); - if (!result.second) - result.first->second += count; - } -} - - -unsigned ContextsCorpus::read_contexts(const string &filename, - BackoffGenerator* backoff_gen_ptr, - bool /*filter_singeltons*/, - bool binary_contexts) { - read_callback_binary_contexts = binary_contexts; - - map<string,int> counts; - //if (filter_singeltons) - { - // cerr << "--- Filtering singleton contexts ---" << endl; - - igzstream in(filename.c_str()); - ContextsLexer::ReadContexts(&in, filter_callback, &counts); - } - - m_num_terms = 0; - m_num_types = 0; - - igzstream in(filename.c_str()); - boost::tuple<ContextsCorpus*, BackoffGenerator*, map<string,int>* > extra_pair(this,backoff_gen_ptr,&counts); - ContextsLexer::ReadContexts(&in, read_callback, &extra_pair); - - //m_num_types = m_dict.max(); - - cerr << "Read backoff with order " << m_backoff->order() << "\n"; - for (int o=0; o<m_backoff->order(); o++) - cerr << " Terms at " << o << " = " << m_backoff->terms_at_level(o) << endl; - //cerr << endl; - - int i=0; double av_freq=0; - for (map<string,int>::const_iterator it=counts.begin(); it != counts.end(); ++it, ++i) { - WordID id = m_dict.Convert(it->first); - m_context_counts[id] = it->second; - av_freq += it->second; - } - cerr << " Average term frequency = " << av_freq / (double) i << endl; - - return m_documents.size(); -} diff --git a/gi/pyp-topics/src/contexts_corpus.hh b/gi/pyp-topics/src/contexts_corpus.hh deleted file mode 100644 index 2527f655..00000000 --- a/gi/pyp-topics/src/contexts_corpus.hh +++ /dev/null @@ -1,90 +0,0 @@ -#ifndef _CONTEXTS_CORPUS_HH -#define _CONTEXTS_CORPUS_HH - -#include <vector> -#include <string> -#include <map> -#include <tr1/unordered_map> - -#include <boost/ptr_container/ptr_vector.hpp> - -#include "corpus.hh" -#include "contexts_lexer.h" -#include "dict.h" - - -class BackoffGenerator { -public: - virtual ContextsLexer::Context - operator()(const ContextsLexer::Context& c) = 0; - -protected: - ContextsLexer::Context strip_edges(const ContextsLexer::Context& c) { - if (c.size() <= 1) return ContextsLexer::Context(); - assert(c.size() % 2 == 1); - return ContextsLexer::Context(c.begin() + 1, c.end() - 1); - } -}; - -class NullBackoffGenerator : public BackoffGenerator { - virtual ContextsLexer::Context - operator()(const ContextsLexer::Context&) - { return ContextsLexer::Context(); } -}; - -class SimpleBackoffGenerator : public BackoffGenerator { - virtual ContextsLexer::Context - operator()(const ContextsLexer::Context& c) { - if (c.size() <= 3) - return ContextsLexer::Context(); - return strip_edges(c); - } -}; - - -//////////////////////////////////////////////////////////////// -// ContextsCorpus -//////////////////////////////////////////////////////////////// - -class ContextsCorpus : public Corpus { - friend void read_callback(const ContextsLexer::PhraseContextsType&, void*); - -public: - ContextsCorpus() : m_backoff(new TermBackoff) {} - virtual ~ContextsCorpus() {} - - virtual unsigned read_contexts(const std::string &filename, - BackoffGenerator* backoff_gen=0, - bool filter_singeltons=false, - bool binary_contexts=false); - - TermBackoffPtr backoff_index() { - return m_backoff; - } - - std::vector<std::string> context2string(const WordID& id) const { - std::vector<std::string> res; - assert (id >= 0); - m_dict.AsVector(id, &res); - return res; - } - - virtual int context_count(const WordID& id) const { - return m_context_counts.find(id)->second; - } - - - const std::string& key(const int& i) const { - return m_keys.at(i); - } - - const Dict& dict() const { return m_dict; } - -protected: - TermBackoffPtr m_backoff; - Dict m_dict; - std::vector<std::string> m_keys; - std::tr1::unordered_map<int,int> m_context_counts; -}; - -#endif // _CONTEXTS_CORPUS_HH diff --git a/gi/pyp-topics/src/contexts_lexer.h b/gi/pyp-topics/src/contexts_lexer.h deleted file mode 100644 index 66004990..00000000 --- a/gi/pyp-topics/src/contexts_lexer.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef _CONTEXTS_LEXER_H_ -#define _CONTEXTS_LEXER_H_ - -#include <iostream> -#include <vector> -#include <string> - -#include "dict.h" - -struct ContextsLexer { - typedef std::vector<std::string> Context; - struct PhraseContextsType { - std::string phrase; - std::vector<Context> contexts; - std::vector< std::pair<int,int> > counts; - }; - - typedef void (*ContextsCallback)(const PhraseContextsType& new_contexts, void* extra); - static void ReadContexts(std::istream* in, ContextsCallback func, void* extra); -}; - -#endif diff --git a/gi/pyp-topics/src/contexts_lexer.l b/gi/pyp-topics/src/contexts_lexer.l deleted file mode 100644 index 64cd7ca3..00000000 --- a/gi/pyp-topics/src/contexts_lexer.l +++ /dev/null @@ -1,113 +0,0 @@ -%{ -#include "contexts_lexer.h" - -#include <string> -#include <iostream> -#include <sstream> -#include <cstring> -#include <cassert> -#include <algorithm> - -int lex_line = 0; -std::istream* contextslex_stream = NULL; -ContextsLexer::ContextsCallback contexts_callback = NULL; -void* contexts_callback_extra = NULL; - -#undef YY_INPUT -#define YY_INPUT(buf, result, max_size) (result = contextslex_stream->read(buf, max_size).gcount()) - -#define YY_SKIP_YYWRAP 1 -int num_phrases = 0; -int yywrap() { return 1; } - -#define MAX_TOKEN_SIZE 255 -std::string contextslex_tmp_token(MAX_TOKEN_SIZE, '\0'); -ContextsLexer::PhraseContextsType current_contexts; - -#define MAX_CONTEXT_SIZE 255 -//std::string tmp_context[MAX_CONTEXT_SIZE]; -ContextsLexer::Context tmp_context; - - -void contextslex_reset() { - current_contexts.phrase.clear(); - current_contexts.contexts.clear(); - current_contexts.counts.clear(); - tmp_context.clear(); -} - -%} - -INT [\-+]?[0-9]+|inf|[\-+]inf - -%x CONTEXT COUNT COUNT_END -%% - -<INITIAL>[^\t]+ { - contextslex_reset(); - current_contexts.phrase.assign(yytext, yyleng); - BEGIN(CONTEXT); - } -<INITIAL>\t { - ; - } - -<INITIAL,CONTEXT,COUNT>\n { - std::cerr << "ERROR: contexts_lexer.l: unexpected newline while trying to read phrase|context|count." << std::endl; - abort(); - } - -<CONTEXT>\|\|\| { - current_contexts.contexts.push_back(tmp_context); - tmp_context.clear(); - BEGIN(COUNT); - } -<CONTEXT>[^ \t]+ { - contextslex_tmp_token.assign(yytext, yyleng); - tmp_context.push_back(contextslex_tmp_token); - } -<CONTEXT>[ \t]+ { ; } - -<COUNT>[ \t]+ { ; } -<COUNT>C={INT} { - current_contexts.counts.push_back(std::make_pair(current_contexts.counts.size(), atoi(yytext+2))); - BEGIN(COUNT_END); - } -<COUNT>. { - std::cerr << "ERROR: contexts_lexer.l: unexpected content while reading count." << std::endl; - abort(); - } - -<COUNT_END>[ \t]+ { ; } -<COUNT_END>\|\|\| { - BEGIN(CONTEXT); - } -<COUNT_END>\n { - //std::cerr << "READ:" << current_contexts.phrase << " with " << current_contexts.contexts.size() - // << " contexts, and " << current_contexts.counts.size() << " counts." << std::endl; - std::sort(current_contexts.counts.rbegin(), current_contexts.counts.rend()); - - contexts_callback(current_contexts, contexts_callback_extra); - current_contexts.phrase.clear(); - current_contexts.contexts.clear(); - current_contexts.counts.clear(); - BEGIN(INITIAL); - } -<COUNT_END>. { - contextslex_tmp_token.assign(yytext, yyleng); - std::cerr << "ERROR: contexts_lexer.l: unexpected content while looking for ||| closing count." << std::endl; - abort(); - } - -%% - -#include "filelib.h" - -void ContextsLexer::ReadContexts(std::istream* in, ContextsLexer::ContextsCallback func, void* extra) { - lex_line = 1; - contextslex_stream = in; - contexts_callback_extra = extra, - contexts_callback = func; - yylex(); -} - diff --git a/gi/pyp-topics/src/corpus.cc b/gi/pyp-topics/src/corpus.cc deleted file mode 100644 index f182381f..00000000 --- a/gi/pyp-topics/src/corpus.cc +++ /dev/null @@ -1,104 +0,0 @@ -#include <sstream> -#include <iostream> -#include <set> - -#include "corpus.hh" -#include "gzstream.hh" - -using namespace std; - -////////////////////////////////////////////////// -// Corpus -////////////////////////////////////////////////// - -Corpus::Corpus() : m_num_terms(0), m_num_types(0) {} - -unsigned Corpus::read(const std::string &filename) { - m_num_terms = 0; - m_num_types = 0; - std::set<int> seen_types; - - igzstream in(filename.c_str()); - - string buf; - int token; - unsigned doc_count=0; - while (getline(in, buf)) { - Document* doc(new Document()); - istringstream ss(buf); - - ss >> token; // the number of unique terms - - char delimeter; - int count; - while(ss >> token >> delimeter >> count) { - for (int i=0; i<count; ++i) - doc->push_back(token); - m_num_terms += count; - seen_types.insert(token); - } - - m_documents.push_back(doc); - doc_count++; - } - - m_num_types = seen_types.size(); - - return doc_count; -} - - -////////////////////////////////////////////////// -// TestCorpus -////////////////////////////////////////////////// - -TestCorpus::TestCorpus() {} - -void TestCorpus::read(const std::string &filename) { - igzstream in(filename.c_str()); - - string buf; - Term term; - DocumentId doc; - char delimeter; - while (getline(in, buf)) { - DocumentTerms* line(new DocumentTerms()); - istringstream ss(buf); - - while(ss >> doc >> delimeter >> term) - line->push_back(DocumentTerm(doc, term)); - - m_lines.push_back(line); - } -} - -////////////////////////////////////////////////// -// TermBackoff -////////////////////////////////////////////////// - -void TermBackoff::read(const std::string &filename) { - igzstream in(filename.c_str()); - - string buf; - int num_terms; - getline(in, buf); - istringstream ss(buf); - ss >> num_terms >> m_backoff_order; - - m_dict.resize(num_terms, -1); - for (int i=0; i<m_backoff_order; ++i) { - int count; ss >> count; - m_terms_at_order.push_back(count); - } - - Term term, backoff; - while (getline(in, buf)) { - istringstream ss(buf); - ss >> term >> backoff; - - assert(term < num_terms); - assert(term >= 0); - - m_dict[term] = backoff; - } -} diff --git a/gi/pyp-topics/src/corpus.hh b/gi/pyp-topics/src/corpus.hh deleted file mode 100644 index 2aa03527..00000000 --- a/gi/pyp-topics/src/corpus.hh +++ /dev/null @@ -1,133 +0,0 @@ -#ifndef _CORPUS_HH -#define _CORPUS_HH - -#include <vector> -#include <string> -#include <map> -#include <limits> - -#include <boost/shared_ptr.hpp> -#include <boost/ptr_container/ptr_vector.hpp> - -//////////////////////////////////////////////////////////////// -// Corpus -//////////////////////////////////////////////////////////////// -typedef int Term; - -typedef std::vector<Term> Document; -typedef std::vector<Term> Terms; - -class Corpus { -public: - typedef boost::ptr_vector<Document>::const_iterator const_iterator; - -public: - Corpus(); - virtual ~Corpus() {} - - virtual unsigned read(const std::string &filename); - - const_iterator begin() const { return m_documents.begin(); } - const_iterator end() const { return m_documents.end(); } - - const Document& at(size_t i) const { return m_documents.at(i); } - - int num_documents() const { return m_documents.size(); } - int num_terms() const { return m_num_terms; } - int num_types() const { return m_num_types; } - - virtual int context_count(const int&) const { - return std::numeric_limits<int>::max(); - } - -protected: - int m_num_terms, m_num_types; - boost::ptr_vector<Document> m_documents; -}; - -typedef int DocumentId; -struct DocumentTerm { - DocumentTerm(DocumentId d, Term t) : term(t), doc(d) {} - Term term; - DocumentId doc; -}; -typedef std::vector<DocumentTerm> DocumentTerms; - -class TestCorpus { -public: - typedef boost::ptr_vector<DocumentTerms>::const_iterator const_iterator; - -public: - TestCorpus(); - ~TestCorpus() {} - - void read(const std::string &filename); - - const_iterator begin() const { return m_lines.begin(); } - const_iterator end() const { return m_lines.end(); } - - int num_instances() const { return m_lines.size(); } - -protected: - boost::ptr_vector<DocumentTerms> m_lines; -}; - -class TermBackoff { -public: - typedef std::vector<Term> dictionary_type; - typedef dictionary_type::const_iterator const_iterator; - const static int NullBackoff=-1; - -public: - TermBackoff() { order(1); } - ~TermBackoff() {} - - void read(const std::string &filename); - - const_iterator begin() const { return m_dict.begin(); } - const_iterator end() const { return m_dict.end(); } - - const Term& operator[](const Term& t) const { - assert(t < static_cast<int>(m_dict.size())); - return m_dict[t]; - } - - Term& operator[](const Term& t) { - if (t >= static_cast<int>(m_dict.size())) - m_dict.resize(t+1, -1); - return m_dict[t]; - } - - bool has_backoff(const Term& t) { - return t >= 0 && t < static_cast<int>(m_dict.size()) && m_dict[t] >= 0; - } - - int order() const { return m_backoff_order; } - void order(int o) { - if (o >= (int)m_terms_at_order.size()) - m_terms_at_order.resize(o, 0); - m_backoff_order = o; - } - -// int levels() const { return m_terms_at_order.size(); } - bool is_null(const Term& term) const { return term < 0; } - int terms_at_level(int level) const { - assert (level < (int)m_terms_at_order.size()); - return m_terms_at_order.at(level); - } - - int& terms_at_level(int level) { - assert (level < (int)m_terms_at_order.size()); - return m_terms_at_order.at(level); - } - - int size() const { return m_dict.size(); } - -protected: - dictionary_type m_dict; - int m_backoff_order; - std::vector<int> m_terms_at_order; -}; -typedef boost::shared_ptr<TermBackoff> TermBackoffPtr; - -#endif // _CORPUS_HH diff --git a/gi/pyp-topics/src/gammadist.c b/gi/pyp-topics/src/gammadist.c deleted file mode 100644 index 4e260db8..00000000 --- a/gi/pyp-topics/src/gammadist.c +++ /dev/null @@ -1,247 +0,0 @@ -/* gammadist.c -- computes probability of samples under / produces samples from a Gamma distribution - * - * Mark Johnson, 22nd March 2008 - * - * WARNING: you need to set the flag -std=c99 to compile - * - * gammavariate() was translated from random.py in Python library - * - * The Gamma distribution is: - * - * Gamma(x | alpha, beta) = pow(x/beta, alpha-1) * exp(-x/beta) / (gamma(alpha)*beta) - * - * shape parameter alpha > 0 (also called c), scale parameter beta > 0 (also called s); - * mean is alpha*beta, variance is alpha*beta**2 - * - * Note that many parameterizations of the Gamma function are in terms of an _inverse_ - * scale parameter beta, which is the inverse of the beta given here. - * - * To define a main() that tests the routines, uncomment the following #define: - */ -/* #define GAMMATEST */ - -#include <assert.h> -#include <math.h> - -#include "gammadist.h" -#include "mt19937ar.h" - -/* gammadist() returns the probability density of x under a Gamma(alpha,beta) - * distribution - */ - -long double gammadist(long double x, long double alpha, long double beta) { - assert(alpha > 0); - assert(beta > 0); - return pow(x/beta, alpha-1) * exp(-x/beta) / (tgamma(alpha)*beta); -} - -/* lgammadist() returns the log probability density of x under a Gamma(alpha,beta) - * distribution - */ - -long double lgammadist(long double x, long double alpha, long double beta) { - assert(alpha > 0); - assert(beta > 0); - return (alpha-1)*log(x) - alpha*log(beta) - x/beta - lgamma(alpha); -} - -/* This definition of gammavariate is from Python code in - * the Python random module. - */ - -long double gammavariate(long double alpha, long double beta) { - - assert(alpha > 0); - assert(beta > 0); - - if (alpha > 1.0) { - - /* Uses R.C.H. Cheng, "The generation of Gamma variables with - non-integral shape parameters", Applied Statistics, (1977), 26, - No. 1, p71-74 */ - - long double ainv = sqrt(2.0 * alpha - 1.0); - long double bbb = alpha - log(4.0); - long double ccc = alpha + ainv; - - while (1) { - long double u1 = mt_genrand_real3(); - if (u1 > 1e-7 || u1 < 0.9999999) { - long double u2 = 1.0 - mt_genrand_real3(); - long double v = log(u1/(1.0-u1))/ainv; - long double x = alpha*exp(v); - long double z = u1*u1*u2; - long double r = bbb+ccc*v-x; - if (r + (1.0+log(4.5)) - 4.5*z >= 0.0 || r >= log(z)) - return x * beta; - } - } - } - else if (alpha == 1.0) { - long double u = mt_genrand_real3(); - while (u <= 1e-7) - u = mt_genrand_real3(); - return -log(u) * beta; - } - else { - /* alpha is between 0 and 1 (exclusive) - Uses ALGORITHM GS of Statistical Computing - Kennedy & Gentle */ - - while (1) { - long double u = mt_genrand_real3(); - long double b = (exp(1) + alpha)/exp(1); - long double p = b*u; - long double x = (p <= 1.0) ? pow(p, 1.0/alpha) : -log((b-p)/alpha); - long double u1 = mt_genrand_real3(); - if (! (((p <= 1.0) && (u1 > exp(-x))) || - ((p > 1.0) && (u1 > pow(x, alpha - 1.0))))) - return x * beta; - } - } -} - -/* betadist() returns the probability density of x under a Beta(alpha,beta) - * distribution. - */ - -long double betadist(long double x, long double alpha, long double beta) { - assert(x >= 0); - assert(x <= 1); - assert(alpha > 0); - assert(beta > 0); - return pow(x,alpha-1)*pow(1-x,beta-1)*tgamma(alpha+beta)/(tgamma(alpha)*tgamma(beta)); -} - -/* lbetadist() returns the log probability density of x under a Beta(alpha,beta) - * distribution. - */ - -long double lbetadist(long double x, long double alpha, long double beta) { - assert(x > 0); - assert(x < 1); - assert(alpha > 0); - assert(beta > 0); - return (alpha-1)*log(x)+(beta-1)*log(1-x)+lgamma(alpha+beta)-lgamma(alpha)-lgamma(beta); -} - -/* betavariate() generates a sample from a Beta distribution with - * parameters alpha and beta. - * - * 0 < alpha < 1, 0 < beta < 1, mean is alpha/(alpha+beta) - */ - -long double betavariate(long double alpha, long double beta) { - long double x = gammavariate(alpha, 1); - long double y = gammavariate(beta, 1); - return x/(x+y); -} - -#ifdef GAMMATEST -#include <stdio.h> - -int main(int argc, char **argv) { - int iteration, niterations = 1000; - - for (iteration = 0; iteration < niterations; ++iteration) { - long double alpha = 100*mt_genrand_real3(); - long double gv = gammavariate(alpha, 1); - long double pgv = gammadist(gv, alpha, 1); - long double pgvl = exp(lgammadist(gv, alpha, 1)); - fprintf(stderr, "iteration = %d, gammavariate(%lg,1) = %lg, gammadist(%lg,%lg,1) = %lg, exp(lgammadist(%lg,%lg,1) = %lg\n", - iteration, alpha, gv, gv, alpha, pgv, gv, alpha, pgvl); - } - return 0; -} - -#endif /* GAMMATEST */ - - -/* Other routines I tried, but which weren't as good as the ones above */ - -#if 0 - -/*! gammavariate() returns samples from a Gamma distribution - *! where alpha is the shape parameter and beta is the scale - *! parameter, using the algorithm described on p. 94 of - *! Gentle (1998) Random Number Generation and Monte Carlo Methods, - *! Springer. - */ - -long double gammavariate(long double alpha) { - - assert(alpha > 0); - - if (alpha > 1.0) { - while (1) { - long double u1 = mt_genrand_real3(); - long double u2 = mt_genrand_real3(); - long double v = (alpha - 1/(6*alpha))*u1/(alpha-1)*u2; - if (2*(u2-1)/(alpha-1) + v + 1/v <= 2 - || 2*log(u2)/(alpha-1) - log(v) + v <= 1) - return (alpha-1)*v; - } - } else if (alpha < 1.0) { - while (1) { - long double t = 0.07 + 0.75*sqrt(1-alpha); - long double b = alpha + exp(-t)*alpha/t; - long double u1 = mt_genrand_real3(); - long double u2 = mt_genrand_real3(); - long double v = b*u1; - if (v <= 1) { - long double x = t*pow(v, 1/alpha); - if (u2 <= (2 - x)/(2 + x)) - return x; - if (u2 <= exp(-x)) - return x; - } - else { - long double x = log(t*(b-v)/alpha); - long double y = x/t; - if (u2*(alpha + y*(1-alpha)) <= 1) - return x; - if (u2 <= pow(y,alpha-1)) - return x; - } - } - } - else - return -log(mt_genrand_real3()); -} - - -/*! gammavariate() returns a deviate distributed as a gamma - *! distribution of order alpha, beta, i.e., a waiting time to the alpha'th - *! event in a Poisson process of unit mean. - *! - *! Code from Numerical Recipes - */ - -long double nr_gammavariate(long double ia) { - int j; - long double am,e,s,v1,v2,x,y; - assert(ia > 0); - if (ia < 10) { - x=1.0; - for (j=1;j<=ia;j++) - x *= mt_genrand_real3(); - x = -log(x); - } else { - do { - do { - do { - v1=mt_genrand_real3(); - v2=2.0*mt_genrand_real3()-1.0; - } while (v1*v1+v2*v2 > 1.0); - y=v2/v1; - am=ia-1; - s=sqrt(2.0*am+1.0); - x=s*y+am; - } while (x <= 0.0); - e=(1.0+y*y)*exp(am*log(x/am)-s*y); - } while (mt_genrand_real3() > e); - } - return x; -} - -#endif diff --git a/gi/pyp-topics/src/gammadist.h b/gi/pyp-topics/src/gammadist.h deleted file mode 100644 index b6ad6c40..00000000 --- a/gi/pyp-topics/src/gammadist.h +++ /dev/null @@ -1,72 +0,0 @@ -/* gammadist.h -- computes probability of samples under / produces samples from a Gamma distribution - * - * Mark Johnson, 22nd March 2008 - * - * gammavariate() was translated from random.py in Python library - * - * The Gamma distribution is: - * - * Gamma(x | alpha, beta) = pow(x/beta, alpha-1) * exp(-x/beta) / (gamma(alpha)*beta) - * - * shape parameter alpha > 0 (also called c), scale parameter beta > 0 (also called s); - * mean is alpha*beta, variance is alpha*beta**2 - * - * Note that many parameterizations of the Gamma function are in terms of an _inverse_ - * scale parameter beta, which is the inverse of the beta given here. - */ - -#ifndef GAMMADIST_H -#define GAMMADIST_H - -#ifdef __cplusplus -extern "C" { -#endif - - /* gammadist() returns the probability density of x under a Gamma(alpha,beta) - * distribution - */ - - long double gammadist(long double x, long double alpha, long double beta); - - /* lgammadist() returns the log probability density of x under a Gamma(alpha,beta) - * distribution - */ - - long double lgammadist(long double x, long double alpha, long double beta); - - /* gammavariate() generates samples from a Gamma distribution - * conditioned on the parameters alpha and beta. - * - * alpha > 0, beta > 0, mean is alpha*beta, variance is alpha*beta**2 - * - * Warning: a few older sources define the gamma distribution in terms - * of alpha > -1.0 - */ - - long double gammavariate(long double alpha, long double beta); - - /* betadist() returns the probability density of x under a Beta(alpha,beta) - * distribution. - */ - - long double betadist(long double x, long double alpha, long double beta); - - /* lbetadist() returns the log probability density of x under a Beta(alpha,beta) - * distribution. - */ - - long double lbetadist(long double x, long double alpha, long double beta); - - /* betavariate() generates a sample from a Beta distribution with - * parameters alpha and beta. - * - * 0 < alpha < 1, 0 < beta < 1, mean is alpha/(alpha+beta) - */ - - long double betavariate(long double alpha, long double beta); - -#ifdef __cplusplus -}; -#endif - -#endif /* GAMMADIST_H */ diff --git a/gi/pyp-topics/src/gzstream.cc b/gi/pyp-topics/src/gzstream.cc deleted file mode 100644 index 7c4d3a12..00000000 --- a/gi/pyp-topics/src/gzstream.cc +++ /dev/null @@ -1,165 +0,0 @@ -// ============================================================================ -// gzstream, C++ iostream classes wrapping the zlib compression library. -// Copyright (C) 2001 Deepak Bandyopadhyay, Lutz Kettner -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// ============================================================================ -// -// File : gzstream.C -// Revision : $Revision: 1.1 $ -// Revision_date : $Date: 2006/03/30 04:05:52 $ -// Author(s) : Deepak Bandyopadhyay, Lutz Kettner -// -// Standard streambuf implementation following Nicolai Josuttis, "The -// Standard C++ Library". -// ============================================================================ - -#include "gzstream.hh" -#include <iostream> -#include <string.h> // for memcpy - -#ifdef GZSTREAM_NAMESPACE -namespace GZSTREAM_NAMESPACE { -#endif - -// ---------------------------------------------------------------------------- -// Internal classes to implement gzstream. See header file for user classes. -// ---------------------------------------------------------------------------- - -// -------------------------------------- -// class gzstreambuf: -// -------------------------------------- - -gzstreambuf* gzstreambuf::open( const char* name, int open_mode) { - if ( is_open()) - return (gzstreambuf*)0; - mode = open_mode; - // no append nor read/write mode - if ((mode & std::ios::ate) || (mode & std::ios::app) - || ((mode & std::ios::in) && (mode & std::ios::out))) - return (gzstreambuf*)0; - char fmode[10]; - char* fmodeptr = fmode; - if ( mode & std::ios::in) - *fmodeptr++ = 'r'; - else if ( mode & std::ios::out) - *fmodeptr++ = 'w'; - *fmodeptr++ = 'b'; - *fmodeptr = '\0'; - file = gzopen( name, fmode); - if (file == 0) - return (gzstreambuf*)0; - opened = 1; - return this; -} - -gzstreambuf * gzstreambuf::close() { - if ( is_open()) { - sync(); - opened = 0; - if ( gzclose( file) == Z_OK) - return this; - } - return (gzstreambuf*)0; -} - -int gzstreambuf::underflow() { // used for input buffer only - if ( gptr() && ( gptr() < egptr())) - return * reinterpret_cast<unsigned char *>( gptr()); - - if ( ! (mode & std::ios::in) || ! opened) - return EOF; - // Josuttis' implementation of inbuf - int n_putback = gptr() - eback(); - if ( n_putback > 4) - n_putback = 4; - memcpy( buffer + (4 - n_putback), gptr() - n_putback, n_putback); - - int num = gzread( file, buffer+4, bufferSize-4); - if (num <= 0) // ERROR or EOF - return EOF; - - // reset buffer pointers - setg( buffer + (4 - n_putback), // beginning of putback area - buffer + 4, // read position - buffer + 4 + num); // end of buffer - - // return next character - return * reinterpret_cast<unsigned char *>( gptr()); -} - -int gzstreambuf::flush_buffer() { - // Separate the writing of the buffer from overflow() and - // sync() operation. - int w = pptr() - pbase(); - if ( gzwrite( file, pbase(), w) != w) - return EOF; - pbump( -w); - return w; -} - -int gzstreambuf::overflow( int c) { // used for output buffer only - if ( ! ( mode & std::ios::out) || ! opened) - return EOF; - if (c != EOF) { - *pptr() = c; - pbump(1); - } - if ( flush_buffer() == EOF) - return EOF; - return c; -} - -int gzstreambuf::sync() { - // Changed to use flush_buffer() instead of overflow( EOF) - // which caused improper behavior with std::endl and flush(), - // bug reported by Vincent Ricard. - if ( pptr() && pptr() > pbase()) { - if ( flush_buffer() == EOF) - return -1; - } - return 0; -} - -// -------------------------------------- -// class gzstreambase: -// -------------------------------------- - -gzstreambase::gzstreambase( const char* name, int mode) { - init( &buf); - open( name, mode); -} - -gzstreambase::~gzstreambase() { - buf.close(); -} - -void gzstreambase::open( const char* name, int open_mode) { - if ( ! buf.open( name, open_mode)) - clear( rdstate() | std::ios::badbit); -} - -void gzstreambase::close() { - if ( buf.is_open()) - if ( ! buf.close()) - clear( rdstate() | std::ios::badbit); -} - -#ifdef GZSTREAM_NAMESPACE -} // namespace GZSTREAM_NAMESPACE -#endif - -// ============================================================================ -// EOF // diff --git a/gi/pyp-topics/src/gzstream.hh b/gi/pyp-topics/src/gzstream.hh deleted file mode 100644 index ad9785fd..00000000 --- a/gi/pyp-topics/src/gzstream.hh +++ /dev/null @@ -1,121 +0,0 @@ -// ============================================================================ -// gzstream, C++ iostream classes wrapping the zlib compression library. -// Copyright (C) 2001 Deepak Bandyopadhyay, Lutz Kettner -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// ============================================================================ -// -// File : gzstream.h -// Revision : $Revision: 1.1 $ -// Revision_date : $Date: 2006/03/30 04:05:52 $ -// Author(s) : Deepak Bandyopadhyay, Lutz Kettner -// -// Standard streambuf implementation following Nicolai Josuttis, "The -// Standard C++ Library". -// ============================================================================ - -#ifndef GZSTREAM_H -#define GZSTREAM_H 1 - -// standard C++ with new header file names and std:: namespace -#include <iostream> -#include <fstream> -#include <zlib.h> - -#ifdef GZSTREAM_NAMESPACE -namespace GZSTREAM_NAMESPACE { -#endif - -// ---------------------------------------------------------------------------- -// Internal classes to implement gzstream. See below for user classes. -// ---------------------------------------------------------------------------- - -class gzstreambuf : public std::streambuf { -private: - static const int bufferSize = 47+256; // size of data buff - // totals 512 bytes under g++ for igzstream at the end. - - gzFile file; // file handle for compressed file - char buffer[bufferSize]; // data buffer - char opened; // open/close state of stream - int mode; // I/O mode - - int flush_buffer(); -public: - gzstreambuf() : opened(0) { - setp( buffer, buffer + (bufferSize-1)); - setg( buffer + 4, // beginning of putback area - buffer + 4, // read position - buffer + 4); // end position - // ASSERT: both input & output capabilities will not be used together - } - int is_open() { return opened; } - gzstreambuf* open( const char* name, int open_mode); - gzstreambuf* close(); - ~gzstreambuf() { close(); } - - virtual int overflow( int c = EOF); - virtual int underflow(); - virtual int sync(); -}; - -class gzstreambase : virtual public std::ios { -protected: - gzstreambuf buf; -public: - gzstreambase() { init(&buf); } - gzstreambase( const char* name, int open_mode); - ~gzstreambase(); - void open( const char* name, int open_mode); - void close(); - gzstreambuf* rdbuf() { return &buf; } -}; - -// ---------------------------------------------------------------------------- -// User classes. Use igzstream and ogzstream analogously to ifstream and -// ofstream respectively. They read and write files based on the gz* -// function interface of the zlib. Files are compatible with gzip compression. -// ---------------------------------------------------------------------------- - -class igzstream : public gzstreambase, public std::istream { -public: - igzstream() : std::istream( &buf) {} - igzstream( const char* name, int open_mode = std::ios::in) - : gzstreambase( name, open_mode), std::istream( &buf) {} - gzstreambuf* rdbuf() { return gzstreambase::rdbuf(); } - void open( const char* name, int open_mode = std::ios::in) { - gzstreambase::open( name, open_mode); - } -}; - -class ogzstream : public gzstreambase, public std::ostream { -public: - ogzstream() : std::ostream( &buf) {} - ogzstream( const char* name, int mode = std::ios::out) - : gzstreambase( name, mode), std::ostream( &buf) {} - gzstreambuf* rdbuf() { return gzstreambase::rdbuf(); } - void open( const char* name, int open_mode = std::ios::out) { - gzstreambase::open( name, open_mode); - } -}; - -#ifdef GZSTREAM_NAMESPACE -} // namespace GZSTREAM_NAMESPACE -#endif - -#endif // GZSTREAM_H -// ============================================================================ -// EOF // - diff --git a/gi/pyp-topics/src/log_add.h b/gi/pyp-topics/src/log_add.h deleted file mode 100644 index e0620c5a..00000000 --- a/gi/pyp-topics/src/log_add.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef log_add_hh -#define log_add_hh - -#include <limits> -#include <iostream> -#include <cassert> -#include <cmath> - -template <typename T> -struct Log -{ - static T zero() { return -std::numeric_limits<T>::infinity(); } - - static T add(T l1, T l2) - { - if (l1 == zero()) return l2; - if (l1 > l2) - return l1 + std::log(1 + exp(l2 - l1)); - else - return l2 + std::log(1 + exp(l1 - l2)); - } - - static T subtract(T l1, T l2) - { - //std::assert(l1 >= l2); - return l1 + log(1 - exp(l2 - l1)); - } -}; - -#endif diff --git a/gi/pyp-topics/src/macros.Linux b/gi/pyp-topics/src/macros.Linux deleted file mode 100644 index 7c6e7fa7..00000000 --- a/gi/pyp-topics/src/macros.Linux +++ /dev/null @@ -1,18 +0,0 @@ -CC = /home/pblunsom/software/bin/mpicc -CXX = /home/pblunsom/software/bin/mpicxx -LD = /home/pblunsom/software/bin/mpicxx -FC = /home/pblunsom/software/bin/mpif77 - -SOFTWARE_DIR=/export/ws10smt/software - -CXXFLAGS = -Wall -I${SOFTWARE_DIR}/include -CFLAGS = -Wall -I${SOFTWARE_DIR}/include -FFLAGS = -Wall -LDFLAGS = -lm -lz -L${SOFTWARE_DIR}/lib \ - -lboost_program_options -lboost_mpi -lboost_serialization \ - -lboost_regex -L../../../decoder -lcdec - -FFLAGS += -g -O6 -march=native -CFLAGS += -g -O6 -march=native -CXXFLAGS += -g -O6 -march=native -LDFLAGS += -g -O6 -march=native diff --git a/gi/pyp-topics/src/makefile.darwin b/gi/pyp-topics/src/makefile.darwin deleted file mode 100644 index af608fd8..00000000 --- a/gi/pyp-topics/src/makefile.darwin +++ /dev/null @@ -1,15 +0,0 @@ -CC = /usr/bin/gcc -CXX = /usr/bin/g++ -LD = /usr/bin/g++ -FC=/usr/bin/g77 - -ARCH=i686-m64 -CXXFLAGS = -m64 -Wall -I/Users/pblunsom/packages/include -CFLAGS = -m64 -Wall -I/Users/pblunsom/packages/include -FFLAGS = -m64 -Wall -LDFLAGS = -L/Users/pblunsom/packages/lib -lboost_program_options -lm -lz - -FFLAGS += -g -O3 -funroll-loops #-pg -CFLAGS += -g -O3 -funroll-loops #-pg -CXXFLAGS += -g -O3 -funroll-loops #-pg -LDFLAGS += -g -O3 -funroll-loops #-pg diff --git a/gi/pyp-topics/src/makefile.depend b/gi/pyp-topics/src/makefile.depend deleted file mode 100644 index 9b8e306c..00000000 --- a/gi/pyp-topics/src/makefile.depend +++ /dev/null @@ -1,4042 +0,0 @@ -contexts_corpus.o: contexts_corpus.cc contexts_corpus.hh \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_vector.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_sequence_adapter.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/reversible_ptr_container.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/assert.hpp \ - /home/pblunsom/packages/include/boost/config.hpp \ - /home/pblunsom/packages/include/boost/config/user.hpp \ - /home/pblunsom/packages/include/boost/config/select_compiler_config.hpp \ - /home/pblunsom/packages/include/boost/config/compiler/gcc.hpp \ - /home/pblunsom/packages/include/boost/config/select_stdlib_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/utility.hpp \ - /home/pblunsom/packages/include/boost/config/stdlib/libstdcpp3.hpp \ - /home/pblunsom/packages/include/boost/config/select_platform_config.hpp \ - /home/pblunsom/packages/include/boost/config/platform/linux.hpp \ - /home/pblunsom/packages/include/boost/config/posix_features.hpp \ - /home/pblunsom/packages/include/boost/config/suffix.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/scoped_deleter.hpp \ - /home/pblunsom/packages/include/boost/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/checked_delete.hpp \ - /home/pblunsom/packages/include/boost/detail/workaround.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/operator_bool.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/static_move_ptr.hpp \ - /home/pblunsom/packages/include/boost/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/detail/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/broken_compiler_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_support.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/gcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/workaround.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ctps.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/cv_traits_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/template_arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/int.hpp \ - /home/pblunsom/packages/include/boost/mpl/int_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/adl_barrier.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/adl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/intel.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nttp_decl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/nttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/integral_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/static_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/static_cast.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/config.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/params.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bool.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/error.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/auto_rec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/eat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/inc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/inc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/overload_resolution.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_empty.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/type_traits/intrinsics.hpp \ - /home/pblunsom/packages/include/boost/type_traits/config.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_same.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/yes_no_type.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_array.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/ice.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_or.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_and.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_not.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_eq.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_arithmetic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_integral.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_float.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_void.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_abstract.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_class.hpp \ - /home/pblunsom/packages/include/boost/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/call_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_function_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_mem_fun_pointer_impl.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/default_deleter.hpp \ - /home/pblunsom/packages/include/boost/mpl/if.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/value_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/integral.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/eti.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/void_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_arity_param.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/dtp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/def_params_tail.hpp \ - /home/pblunsom/packages/include/boost/mpl/limits/arity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/and.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/add.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/dec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/adt.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/check.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/compl.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/detail/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/sub.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_bounds.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/mpl/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/use_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nested_type_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/include_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/compiler.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/stringize.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/identity.hpp \ - /home/pblunsom/packages/include/boost/utility/enable_if.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/move.hpp \ - /home/pblunsom/packages/include/boost/static_assert.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/clone_allocator.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/nullable.hpp \ - /home/pblunsom/packages/include/boost/mpl/eval_if.hpp \ - /home/pblunsom/packages/include/boost/range/functions.hpp \ - /home/pblunsom/packages/include/boost/range/begin.hpp \ - /home/pblunsom/packages/include/boost/range/config.hpp \ - /home/pblunsom/packages/include/boost/range/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/mutable_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/detail/extract_optional_type.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/const_iterator.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_const.hpp \ - /home/pblunsom/packages/include/boost/range/end.hpp \ - /home/pblunsom/packages/include/boost/range/detail/implementation_help.hpp \ - /home/pblunsom/packages/include/boost/range/detail/common.hpp \ - /home/pblunsom/packages/include/boost/range/detail/sfinae.hpp \ - /home/pblunsom/packages/include/boost/range/size.hpp \ - /home/pblunsom/packages/include/boost/range/difference_type.hpp \ - /home/pblunsom/packages/include/boost/range/distance.hpp \ - /home/pblunsom/packages/include/boost/range/empty.hpp \ - /home/pblunsom/packages/include/boost/range/rbegin.hpp \ - /home/pblunsom/packages/include/boost/range/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator.hpp \ - /home/pblunsom/packages/include/boost/utility.hpp \ - /home/pblunsom/packages/include/boost/utility/addressof.hpp \ - /home/pblunsom/packages/include/boost/utility/base_from_member.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/rem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat_from_to.hpp \ - /home/pblunsom/packages/include/boost/utility/binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/deduce_d.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mod.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/detail/div_base.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/not.hpp \ - /home/pblunsom/packages/include/boost/next_prior.hpp \ - /home/pblunsom/packages/include/boost/noncopyable.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_adaptor.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_categories.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/not.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/yes_no.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/arrays.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/pp_counter.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arg_typedef.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/placeholders.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_undef.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_facade.hpp \ - /home/pblunsom/packages/include/boost/iterator/interoperable.hpp \ - /home/pblunsom/packages/include/boost/mpl/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/or.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/facade_iterator_category.hpp \ - /home/pblunsom/packages/include/boost/detail/indirect_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_function.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/false_result.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_function_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_pointer.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/enable_if.hpp \ - /home/pblunsom/packages/include/boost/implicit_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pod.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_scalar.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/always.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/type_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc_typename.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_never_true.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/next.hpp \ - /home/pblunsom/packages/include/boost/mpl/next_prior.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/common_name_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/protect.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/void.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_type.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply.hpp \ - /home/pblunsom/packages/include/boost/range/rend.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/indirect_fun.hpp \ - /home/pblunsom/packages/include/boost/utility/result_of.hpp \ - /home/pblunsom/packages/include/boost/type.hpp \ - /home/pblunsom/packages/include/boost/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/library.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/div.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mul.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/data.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/not_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_z.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/limits.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/assert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/line.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/def.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/apply.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_unary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/expand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/intercept.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/local.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/self.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/append.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/at.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/detail/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitnor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitxor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/nor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/or.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/xor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_r.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_a_default.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_defaults.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/max.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/min.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/detail/split.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/subseq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/iter/forward1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/lower1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/shared.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/upper1.hpp \ - /home/pblunsom/packages/include/boost/utility/detail/result_of_iterate.hpp \ - /home/pblunsom/packages/include/boost/pointee.hpp \ - /home/pblunsom/packages/include/boost/detail/is_incrementable.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/void_ptr_iterator.hpp \ - corpus.hh /home/pblunsom/packages/include/boost/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/memory.hpp \ - /home/pblunsom/packages/include/boost/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/attribute_noreturn.hpp \ - /home/pblunsom/packages/include/boost/exception/exception.hpp \ - /home/pblunsom/packages/include/boost/current_function.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/shared_count.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/bad_weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_has_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp \ - /home/pblunsom/packages/include/boost/detail/sp_typeinfo.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_impl.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_convertible.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_pool.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/yield_k.hpp \ - /home/pblunsom/packages/include/boost/memory_order.hpp contexts_lexer.h \ - ../../../decoder/dict.h \ - /home/pblunsom/packages/include/boost/functional/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash_fwd.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/float_functions.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/cmath.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/limits.hpp \ - /home/pblunsom/packages/include/boost/limits.hpp \ - /home/pblunsom/packages/include/boost/integer/static_log2.hpp \ - /home/pblunsom/packages/include/boost/integer_fwd.hpp \ - /home/pblunsom/packages/include/boost/cstdint.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float_generic.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/extensions.hpp \ - /home/pblunsom/packages/include/boost/detail/container_fwd.hpp \ - ../../../decoder/wordid.h gzstream.hh \ - /home/pblunsom/packages/include/boost/tuple/tuple.hpp \ - /home/pblunsom/packages/include/boost/ref.hpp \ - /home/pblunsom/packages/include/boost/tuple/detail/tuple_basic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/cv_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/function_traits.hpp -contexts_lexer.o: contexts_lexer.cc contexts_lexer.h \ - ../../../decoder/dict.h \ - /home/pblunsom/packages/include/boost/functional/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash_fwd.hpp \ - /home/pblunsom/packages/include/boost/config.hpp \ - /home/pblunsom/packages/include/boost/config/user.hpp \ - /home/pblunsom/packages/include/boost/config/select_compiler_config.hpp \ - /home/pblunsom/packages/include/boost/config/compiler/gcc.hpp \ - /home/pblunsom/packages/include/boost/config/select_stdlib_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/utility.hpp \ - /home/pblunsom/packages/include/boost/config/stdlib/libstdcpp3.hpp \ - /home/pblunsom/packages/include/boost/config/select_platform_config.hpp \ - /home/pblunsom/packages/include/boost/config/platform/linux.hpp \ - /home/pblunsom/packages/include/boost/config/posix_features.hpp \ - /home/pblunsom/packages/include/boost/config/suffix.hpp \ - /home/pblunsom/packages/include/boost/detail/workaround.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/float_functions.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/cmath.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/limits.hpp \ - /home/pblunsom/packages/include/boost/limits.hpp \ - /home/pblunsom/packages/include/boost/integer/static_log2.hpp \ - /home/pblunsom/packages/include/boost/integer_fwd.hpp \ - /home/pblunsom/packages/include/boost/cstdint.hpp \ - /home/pblunsom/packages/include/boost/assert.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float_generic.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/extensions.hpp \ - /home/pblunsom/packages/include/boost/detail/container_fwd.hpp \ - ../../../decoder/wordid.h ../../../decoder/filelib.h \ - ../../../decoder/gzstream.h -corpus.o: corpus.cc corpus.hh \ - /home/pblunsom/packages/include/boost/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/config.hpp \ - /home/pblunsom/packages/include/boost/config/user.hpp \ - /home/pblunsom/packages/include/boost/config/select_compiler_config.hpp \ - /home/pblunsom/packages/include/boost/config/compiler/gcc.hpp \ - /home/pblunsom/packages/include/boost/config/select_stdlib_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/utility.hpp \ - /home/pblunsom/packages/include/boost/config/stdlib/libstdcpp3.hpp \ - /home/pblunsom/packages/include/boost/config/select_platform_config.hpp \ - /home/pblunsom/packages/include/boost/config/platform/linux.hpp \ - /home/pblunsom/packages/include/boost/config/posix_features.hpp \ - /home/pblunsom/packages/include/boost/config/suffix.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/memory.hpp \ - /home/pblunsom/packages/include/boost/assert.hpp \ - /home/pblunsom/packages/include/boost/checked_delete.hpp \ - /home/pblunsom/packages/include/boost/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/attribute_noreturn.hpp \ - /home/pblunsom/packages/include/boost/detail/workaround.hpp \ - /home/pblunsom/packages/include/boost/exception/exception.hpp \ - /home/pblunsom/packages/include/boost/current_function.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/shared_count.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/bad_weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_has_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp \ - /home/pblunsom/packages/include/boost/detail/sp_typeinfo.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_impl.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_convertible.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_pool.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/yield_k.hpp \ - /home/pblunsom/packages/include/boost/memory_order.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/operator_bool.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_vector.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_sequence_adapter.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/reversible_ptr_container.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/scoped_deleter.hpp \ - /home/pblunsom/packages/include/boost/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/static_move_ptr.hpp \ - /home/pblunsom/packages/include/boost/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/detail/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/broken_compiler_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_support.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/gcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/workaround.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ctps.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/cv_traits_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/template_arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/int.hpp \ - /home/pblunsom/packages/include/boost/mpl/int_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/adl_barrier.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/adl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/intel.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nttp_decl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/nttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/integral_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/static_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/static_cast.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/config.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/params.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bool.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/error.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/auto_rec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/eat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/inc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/inc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/overload_resolution.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_empty.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/type_traits/intrinsics.hpp \ - /home/pblunsom/packages/include/boost/type_traits/config.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_same.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/yes_no_type.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_array.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/ice.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_or.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_and.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_not.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_eq.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_arithmetic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_integral.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_float.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_void.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_abstract.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_class.hpp \ - /home/pblunsom/packages/include/boost/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/call_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_function_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_mem_fun_pointer_impl.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/default_deleter.hpp \ - /home/pblunsom/packages/include/boost/mpl/if.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/value_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/integral.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/eti.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/void_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_arity_param.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/dtp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/def_params_tail.hpp \ - /home/pblunsom/packages/include/boost/mpl/limits/arity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/and.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/add.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/dec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/adt.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/check.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/compl.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/detail/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/sub.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_bounds.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/mpl/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/use_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nested_type_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/include_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/compiler.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/stringize.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/identity.hpp \ - /home/pblunsom/packages/include/boost/utility/enable_if.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/move.hpp \ - /home/pblunsom/packages/include/boost/static_assert.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/clone_allocator.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/nullable.hpp \ - /home/pblunsom/packages/include/boost/mpl/eval_if.hpp \ - /home/pblunsom/packages/include/boost/range/functions.hpp \ - /home/pblunsom/packages/include/boost/range/begin.hpp \ - /home/pblunsom/packages/include/boost/range/config.hpp \ - /home/pblunsom/packages/include/boost/range/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/mutable_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/detail/extract_optional_type.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/const_iterator.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_const.hpp \ - /home/pblunsom/packages/include/boost/range/end.hpp \ - /home/pblunsom/packages/include/boost/range/detail/implementation_help.hpp \ - /home/pblunsom/packages/include/boost/range/detail/common.hpp \ - /home/pblunsom/packages/include/boost/range/detail/sfinae.hpp \ - /home/pblunsom/packages/include/boost/range/size.hpp \ - /home/pblunsom/packages/include/boost/range/difference_type.hpp \ - /home/pblunsom/packages/include/boost/range/distance.hpp \ - /home/pblunsom/packages/include/boost/range/empty.hpp \ - /home/pblunsom/packages/include/boost/range/rbegin.hpp \ - /home/pblunsom/packages/include/boost/range/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator.hpp \ - /home/pblunsom/packages/include/boost/utility.hpp \ - /home/pblunsom/packages/include/boost/utility/addressof.hpp \ - /home/pblunsom/packages/include/boost/utility/base_from_member.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/rem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat_from_to.hpp \ - /home/pblunsom/packages/include/boost/utility/binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/deduce_d.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mod.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/detail/div_base.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/not.hpp \ - /home/pblunsom/packages/include/boost/next_prior.hpp \ - /home/pblunsom/packages/include/boost/noncopyable.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_adaptor.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_categories.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/not.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/yes_no.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/arrays.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/pp_counter.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arg_typedef.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/placeholders.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_undef.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_facade.hpp \ - /home/pblunsom/packages/include/boost/iterator/interoperable.hpp \ - /home/pblunsom/packages/include/boost/mpl/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/or.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/facade_iterator_category.hpp \ - /home/pblunsom/packages/include/boost/detail/indirect_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_function.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/false_result.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_function_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_pointer.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/enable_if.hpp \ - /home/pblunsom/packages/include/boost/implicit_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pod.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_scalar.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/always.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/type_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc_typename.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_never_true.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/next.hpp \ - /home/pblunsom/packages/include/boost/mpl/next_prior.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/common_name_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/protect.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/void.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_type.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply.hpp \ - /home/pblunsom/packages/include/boost/range/rend.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/indirect_fun.hpp \ - /home/pblunsom/packages/include/boost/utility/result_of.hpp \ - /home/pblunsom/packages/include/boost/type.hpp \ - /home/pblunsom/packages/include/boost/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/library.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/div.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mul.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/data.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/not_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_z.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/limits.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/assert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/line.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/def.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/apply.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_unary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/expand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/intercept.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/local.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/self.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/append.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/at.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/detail/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitnor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitxor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/nor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/or.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/xor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_r.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_a_default.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_defaults.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/max.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/min.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/detail/split.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/subseq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/iter/forward1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/lower1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/shared.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/upper1.hpp \ - /home/pblunsom/packages/include/boost/utility/detail/result_of_iterate.hpp \ - /home/pblunsom/packages/include/boost/pointee.hpp \ - /home/pblunsom/packages/include/boost/detail/is_incrementable.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/void_ptr_iterator.hpp \ - gzstream.hh -gzstream.o: gzstream.cc gzstream.hh -mpi-pyp-topics.o: mpi-pyp-topics.cc \ - /home/pblunsom/packages/include/boost/mpi/communicator.hpp \ - /home/pblunsom/packages/include/boost/mpi/config.hpp \ - /home/pblunsom/packages/include/mpi.h \ - /home/pblunsom/packages/include/mpio.h \ - /home/pblunsom/packages/include/mpi.h \ - /home/pblunsom/packages/include/mpicxx.h \ - /home/pblunsom/packages/include/boost/config.hpp \ - /home/pblunsom/packages/include/boost/config/user.hpp \ - /home/pblunsom/packages/include/boost/config/select_compiler_config.hpp \ - /home/pblunsom/packages/include/boost/config/compiler/gcc.hpp \ - /home/pblunsom/packages/include/boost/config/select_stdlib_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/utility.hpp \ - /home/pblunsom/packages/include/boost/config/stdlib/libstdcpp3.hpp \ - /home/pblunsom/packages/include/boost/config/select_platform_config.hpp \ - /home/pblunsom/packages/include/boost/config/platform/linux.hpp \ - /home/pblunsom/packages/include/boost/config/posix_features.hpp \ - /home/pblunsom/packages/include/boost/config/suffix.hpp \ - /home/pblunsom/packages/include/boost/config/auto_link.hpp \ - /home/pblunsom/packages/include/boost/mpi/exception.hpp \ - /home/pblunsom/packages/include/boost/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/attribute_noreturn.hpp \ - /home/pblunsom/packages/include/boost/detail/workaround.hpp \ - /home/pblunsom/packages/include/boost/exception/exception.hpp \ - /home/pblunsom/packages/include/boost/current_function.hpp \ - /home/pblunsom/packages/include/boost/optional.hpp \ - /home/pblunsom/packages/include/boost/optional/optional.hpp \ - /home/pblunsom/packages/include/boost/assert.hpp \ - /home/pblunsom/packages/include/boost/type.hpp \ - /home/pblunsom/packages/include/boost/type_traits/alignment_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/intrinsics.hpp \ - /home/pblunsom/packages/include/boost/type_traits/config.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_same.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/template_arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/int.hpp \ - /home/pblunsom/packages/include/boost/mpl/int_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/adl_barrier.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/adl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/intel.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/gcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/workaround.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nttp_decl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/nttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/integral_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/static_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/static_cast.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/config.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/params.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bool.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/error.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/auto_rec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/eat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/inc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/inc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ctps.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/overload_resolution.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_support.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/cv_traits_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/type_with_alignment.hpp \ - /home/pblunsom/packages/include/boost/mpl/if.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/value_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/integral.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/eti.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/void_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_arity_param.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/dtp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/def_params_tail.hpp \ - /home/pblunsom/packages/include/boost/mpl/limits/arity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/and.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/add.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/dec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/adt.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/check.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/compl.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/detail/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/sub.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/detail/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/rem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/append.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pod.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_void.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_scalar.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_arithmetic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_integral.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_float.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_or.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_enum.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_function_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_mem_fun_pointer_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/broken_compiler_spec.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_and.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_not.hpp \ - /home/pblunsom/packages/include/boost/static_assert.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_reference.hpp \ - /home/pblunsom/packages/include/boost/mpl/not.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nested_type_wknd.hpp \ - /home/pblunsom/packages/include/boost/detail/reference_content.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_copy.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_copy.hpp \ - /home/pblunsom/packages/include/boost/mpl/void.hpp \ - /home/pblunsom/packages/include/boost/none.hpp \ - /home/pblunsom/packages/include/boost/none_t.hpp \ - /home/pblunsom/packages/include/boost/utility/compare_pointees.hpp \ - /home/pblunsom/packages/include/boost/optional/optional_fwd.hpp \ - /home/pblunsom/packages/include/boost/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/memory.hpp \ - /home/pblunsom/packages/include/boost/checked_delete.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/shared_count.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/bad_weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_has_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp \ - /home/pblunsom/packages/include/boost/detail/sp_typeinfo.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_impl.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_convertible.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_pool.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/yield_k.hpp \ - /home/pblunsom/packages/include/boost/memory_order.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/operator_bool.hpp \ - /home/pblunsom/packages/include/boost/mpi/datatype.hpp \ - /home/pblunsom/packages/include/boost/mpi/datatype_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/use_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/include_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/compiler.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/stringize.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/and.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/mpi_datatype_cache.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/mpi_datatype_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/oserializer.hpp \ - /home/pblunsom/packages/include/boost/mpl/eval_if.hpp \ - /home/pblunsom/packages/include/boost/mpl/equal_to.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/comparison_op.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/numeric_op.hpp \ - /home/pblunsom/packages/include/boost/mpl/numeric_cast.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/type_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/yes_no.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/arrays.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc_typename.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_never_true.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/numeric_cast_utils.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/forwarding.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_eti_base.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/is_msvc_eti_arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/equal_to.hpp \ - /home/pblunsom/packages/include/boost/mpl/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/mpl/identity.hpp \ - /home/pblunsom/packages/include/boost/serialization/extended_type_info_typeid.hpp \ - /home/pblunsom/packages/include/boost/serialization/static_warning.hpp \ - /home/pblunsom/packages/include/boost/mpl/print.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_polymorphic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_const.hpp \ - /home/pblunsom/packages/include/boost/serialization/singleton.hpp \ - /home/pblunsom/packages/include/boost/noncopyable.hpp \ - /home/pblunsom/packages/include/boost/serialization/force_include.hpp \ - /home/pblunsom/packages/include/boost/serialization/extended_type_info.hpp \ - /home/pblunsom/packages/include/boost/serialization/config.hpp \ - /home/pblunsom/packages/include/boost/config/abi_prefix.hpp \ - /home/pblunsom/packages/include/boost/config/abi_suffix.hpp \ - /home/pblunsom/packages/include/boost/serialization/factory.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/not.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/not_equal.hpp \ - /home/pblunsom/packages/include/boost/serialization/access.hpp \ - /home/pblunsom/packages/include/boost/serialization/pfto.hpp \ - /home/pblunsom/packages/include/boost/serialization/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/serialization/smart_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_and_derived.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_pointer.hpp \ - /home/pblunsom/packages/include/boost/serialization/assume_abstract.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_abstract.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_extent.hpp \ - /home/pblunsom/packages/include/boost/serialization/serialization.hpp \ - /home/pblunsom/packages/include/boost/serialization/strong_typedef.hpp \ - /home/pblunsom/packages/include/boost/operators.hpp \ - /home/pblunsom/packages/include/boost/iterator.hpp \ - /home/pblunsom/packages/include/boost/serialization/nvp.hpp \ - /home/pblunsom/packages/include/boost/serialization/level.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_fundamental.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_array.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_class.hpp \ - /home/pblunsom/packages/include/boost/serialization/level_enum.hpp \ - /home/pblunsom/packages/include/boost/serialization/tracking.hpp \ - /home/pblunsom/packages/include/boost/mpl/greater.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/greater.hpp \ - /home/pblunsom/packages/include/boost/serialization/tracking_enum.hpp \ - /home/pblunsom/packages/include/boost/serialization/type_info_implementation.hpp \ - /home/pblunsom/packages/include/boost/serialization/traits.hpp \ - /home/pblunsom/packages/include/boost/serialization/split_member.hpp \ - /home/pblunsom/packages/include/boost/serialization/base_object.hpp \ - /home/pblunsom/packages/include/boost/serialization/void_cast_fwd.hpp \ - /home/pblunsom/packages/include/boost/serialization/wrapper.hpp \ - /home/pblunsom/packages/include/boost/serialization/version.hpp \ - /home/pblunsom/packages/include/boost/mpl/assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/pp_counter.hpp \ - /home/pblunsom/packages/include/boost/mpl/less.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/less.hpp \ - /home/pblunsom/packages/include/boost/mpl/comparison.hpp \ - /home/pblunsom/packages/include/boost/mpl/not_equal_to.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/not_equal_to.hpp \ - /home/pblunsom/packages/include/boost/mpl/less_equal.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/less_equal.hpp \ - /home/pblunsom/packages/include/boost/serialization/void_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_virtual_base_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_of.hpp \ - /home/pblunsom/packages/include/boost/serialization/array.hpp \ - /home/pblunsom/packages/include/boost/mpl/always.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arg_typedef.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/next.hpp \ - /home/pblunsom/packages/include/boost/mpl/next_prior.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/common_name_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/protect.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_type.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply.hpp \ - /home/pblunsom/packages/include/boost/array.hpp \ - /home/pblunsom/packages/include/boost/swap.hpp \ - /home/pblunsom/packages/include/boost/utility/swap.hpp \ - /home/pblunsom/packages/include/boost/detail/iterator.hpp \ - /home/pblunsom/packages/include/boost/serialization/collection_size_type.hpp \ - /home/pblunsom/packages/include/boost/archive/archive_exception.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/decl.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/abi_prefix.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/abi_suffix.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/basic_archive.hpp \ - /home/pblunsom/packages/include/boost/cstdint.hpp \ - /home/pblunsom/packages/include/boost/integer_traits.hpp \ - /home/pblunsom/packages/include/boost/limits.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/auto_link_archive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_oserializer.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_serializer.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_pointer_oserializer.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/archive_serializer_map.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/check.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/ignore_skeleton_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/common_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/interface_oarchive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/mpi_datatype_primitive.hpp \ - /home/pblunsom/packages/include/boost/serialization/detail/get_data.hpp \ - /home/pblunsom/packages/include/boost/integer.hpp \ - /home/pblunsom/packages/include/boost/integer_fwd.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/register_archive.hpp \ - /home/pblunsom/packages/include/boost/utility/enable_if.hpp \ - /home/pblunsom/packages/include/boost/mpi/packed_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/basic_binary_oarchive.hpp \ - /home/pblunsom/packages/include/boost/serialization/string.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/packed_oprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/allocator.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/binary_buffer_oprimitive.hpp \ - /home/pblunsom/packages/include/boost/serialization/is_bitwise_serializable.hpp \ - /home/pblunsom/packages/include/boost/mpi/packed_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/basic_binary_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/common_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_pointer_iserializer.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/interface_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/iserializer.hpp \ - /home/pblunsom/packages/include/boost/detail/no_exceptions_support.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_new_operator.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/yes_no_type.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_iserializer.hpp \ - /home/pblunsom/packages/include/boost/archive/shared_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/serialization/shared_ptr_132.hpp \ - /home/pblunsom/packages/include/boost/serialization/split_free.hpp \ - /home/pblunsom/packages/include/boost/serialization/detail/shared_ptr_132.hpp \ - /home/pblunsom/packages/include/boost/serialization/detail/shared_count_132.hpp \ - /home/pblunsom/packages/include/boost/detail/lightweight_mutex.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/lightweight_mutex.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/lwm_pthreads.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/packed_iprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/binary_buffer_iprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/skeleton_and_content_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/point_to_point.hpp \ - /home/pblunsom/packages/include/boost/mpi/status.hpp \ - /home/pblunsom/packages/include/boost/mpi/request.hpp timing.h \ - clock_gettime_stub.c mpi-pyp-topics.hh \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_vector.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_sequence_adapter.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/reversible_ptr_container.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/scoped_deleter.hpp \ - /home/pblunsom/packages/include/boost/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/static_move_ptr.hpp \ - /home/pblunsom/packages/include/boost/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/detail/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_empty.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/ice.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_eq.hpp \ - /home/pblunsom/packages/include/boost/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/call_traits.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/default_deleter.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_bounds.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/move.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/clone_allocator.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/nullable.hpp \ - /home/pblunsom/packages/include/boost/range/functions.hpp \ - /home/pblunsom/packages/include/boost/range/begin.hpp \ - /home/pblunsom/packages/include/boost/range/config.hpp \ - /home/pblunsom/packages/include/boost/range/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/mutable_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/detail/extract_optional_type.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_traits.hpp \ - /home/pblunsom/packages/include/boost/range/const_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/end.hpp \ - /home/pblunsom/packages/include/boost/range/detail/implementation_help.hpp \ - /home/pblunsom/packages/include/boost/range/detail/common.hpp \ - /home/pblunsom/packages/include/boost/range/detail/sfinae.hpp \ - /home/pblunsom/packages/include/boost/range/size.hpp \ - /home/pblunsom/packages/include/boost/range/difference_type.hpp \ - /home/pblunsom/packages/include/boost/range/distance.hpp \ - /home/pblunsom/packages/include/boost/range/empty.hpp \ - /home/pblunsom/packages/include/boost/range/rbegin.hpp \ - /home/pblunsom/packages/include/boost/range/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/utility.hpp \ - /home/pblunsom/packages/include/boost/utility/addressof.hpp \ - /home/pblunsom/packages/include/boost/utility/base_from_member.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat_from_to.hpp \ - /home/pblunsom/packages/include/boost/utility/binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/deduce_d.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mod.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/detail/div_base.hpp \ - /home/pblunsom/packages/include/boost/next_prior.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_adaptor.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_categories.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_def.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_undef.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_facade.hpp \ - /home/pblunsom/packages/include/boost/iterator/interoperable.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/facade_iterator_category.hpp \ - /home/pblunsom/packages/include/boost/detail/indirect_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_function.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/false_result.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_function_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/enable_if.hpp \ - /home/pblunsom/packages/include/boost/implicit_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_pointer.hpp \ - /home/pblunsom/packages/include/boost/range/rend.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/indirect_fun.hpp \ - /home/pblunsom/packages/include/boost/utility/result_of.hpp \ - /home/pblunsom/packages/include/boost/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/library.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/div.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mul.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/data.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_z.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/limits.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/assert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/line.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/def.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/apply.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_unary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/expand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/intercept.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/local.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/self.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/at.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitnor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitxor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/nor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/or.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/xor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_r.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_a_default.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_defaults.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/max.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/min.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/detail/split.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/subseq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/iter/forward1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/lower1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/shared.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/upper1.hpp \ - /home/pblunsom/packages/include/boost/utility/detail/result_of_iterate.hpp \ - /home/pblunsom/packages/include/boost/pointee.hpp \ - /home/pblunsom/packages/include/boost/detail/is_incrementable.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/void_ptr_iterator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_real.hpp \ - /home/pblunsom/packages/include/boost/random/detail/config.hpp \ - /home/pblunsom/packages/include/boost/random/variate_generator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_01.hpp \ - /home/pblunsom/packages/include/boost/random/detail/pass_through_engine.hpp \ - /home/pblunsom/packages/include/boost/random/detail/ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/random/detail/disable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/enable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/uniform_int_float.hpp \ - /home/pblunsom/packages/include/boost/random/mersenne_twister.hpp \ - /home/pblunsom/packages/include/boost/random/linear_congruential.hpp \ - /home/pblunsom/packages/include/boost/random/detail/const_mod.hpp \ - /home/pblunsom/packages/include/boost/random/detail/seed.hpp \ - /home/pblunsom/packages/include/boost/random/inversive_congruential.hpp \ - /home/pblunsom/packages/include/boost/random/lagged_fibonacci.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/cmath.hpp \ - /home/pblunsom/packages/include/boost/mpi/environment.hpp mpi-pyp.hh \ - /home/pblunsom/packages/include/boost/tuple/tuple.hpp \ - /home/pblunsom/packages/include/boost/ref.hpp \ - /home/pblunsom/packages/include/boost/tuple/detail/tuple_basic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/cv_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/function_traits.hpp \ - /home/pblunsom/packages/include/boost/serialization/map.hpp \ - /home/pblunsom/packages/include/boost/serialization/utility.hpp \ - /home/pblunsom/packages/include/boost/serialization/collections_save_imp.hpp \ - /home/pblunsom/packages/include/boost/serialization/collections_load_imp.hpp \ - /home/pblunsom/packages/include/boost/serialization/detail/stack_constructor.hpp \ - /home/pblunsom/packages/include/boost/aligned_storage.hpp \ - /home/pblunsom/packages/include/boost/mpi.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/all_gather.hpp \ - /home/pblunsom/packages/include/boost/serialization/vector.hpp \ - /home/pblunsom/packages/include/boost/serialization/collection_traits.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/broadcast.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/gather.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/all_reduce.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/reduce.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/computation_tree.hpp \ - /home/pblunsom/packages/include/boost/mpi/operations.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/all_to_all.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/scatter.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/scan.hpp \ - /home/pblunsom/packages/include/boost/mpi/graph_communicator.hpp \ - /home/pblunsom/packages/include/boost/graph/graph_traits.hpp \ - /home/pblunsom/packages/include/boost/pending/property.hpp \ - /home/pblunsom/packages/include/boost/pending/detail/property.hpp \ - /home/pblunsom/packages/include/boost/type_traits/same_traits.hpp \ - /home/pblunsom/packages/include/boost/graph/properties.hpp \ - /home/pblunsom/packages/include/boost/property_map/property_map.hpp \ - /home/pblunsom/packages/include/boost/pending/cstddef.hpp \ - /home/pblunsom/packages/include/boost/concept_check.hpp \ - /home/pblunsom/packages/include/boost/concept/assert.hpp \ - /home/pblunsom/packages/include/boost/concept/detail/general.hpp \ - /home/pblunsom/packages/include/boost/concept/detail/has_constraints.hpp \ - /home/pblunsom/packages/include/boost/type_traits/conversion_traits.hpp \ - /home/pblunsom/packages/include/boost/concept/usage.hpp \ - /home/pblunsom/packages/include/boost/concept/detail/concept_def.hpp \ - /home/pblunsom/packages/include/boost/concept/detail/concept_undef.hpp \ - /home/pblunsom/packages/include/boost/concept_archetype.hpp \ - /home/pblunsom/packages/include/boost/property_map/vector_property_map.hpp \ - /home/pblunsom/packages/include/boost/graph/property_maps/constant_property_map.hpp \ - /home/pblunsom/packages/include/boost/graph/property_maps/null_property_map.hpp \ - /home/pblunsom/packages/include/boost/iterator/counting_iterator.hpp \ - /home/pblunsom/packages/include/boost/detail/numeric_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_assign.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_assign.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_constructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_constructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_virtual_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_signed.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_unsigned.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_compound.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_floating_point.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_object_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_object.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_stateless.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_union.hpp \ - /home/pblunsom/packages/include/boost/type_traits/rank.hpp \ - /home/pblunsom/packages/include/boost/type_traits/extent.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_all_extents.hpp \ - /home/pblunsom/packages/include/boost/type_traits/aligned_storage.hpp \ - /home/pblunsom/packages/include/boost/type_traits/floating_point_promotion.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_promotion.hpp \ - /home/pblunsom/packages/include/boost/type_traits/promote.hpp \ - /home/pblunsom/packages/include/boost/type_traits/make_unsigned.hpp \ - /home/pblunsom/packages/include/boost/type_traits/make_signed.hpp \ - /home/pblunsom/packages/include/boost/type_traits/decay.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_complex.hpp \ - /home/pblunsom/packages/include/boost/detail/select_type.hpp \ - /home/pblunsom/packages/include/boost/graph/iteration_macros.hpp \ - /home/pblunsom/packages/include/boost/shared_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_array.hpp \ - /home/pblunsom/packages/include/boost/mpi/group.hpp \ - /home/pblunsom/packages/include/boost/mpi/intercommunicator.hpp \ - /home/pblunsom/packages/include/boost/mpi/nonblocking.hpp \ - /home/pblunsom/packages/include/boost/mpi/skeleton_and_content.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/forward_skeleton_iarchive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/forward_skeleton_oarchive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/ignore_iprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/ignore_oprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/content_oarchive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/broadcast_sc.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/communicator_sc.hpp \ - /home/pblunsom/packages/include/boost/mpi/timer.hpp pyp.hh \ - slice-sampler.h log_add.h mt19937ar.h corpus.hh -mpi-train-contexts.o: mpi-train-contexts.cc \ - /home/pblunsom/packages/include/boost/program_options/parsers.hpp \ - /home/pblunsom/packages/include/boost/program_options/config.hpp \ - /home/pblunsom/packages/include/boost/config.hpp \ - /home/pblunsom/packages/include/boost/config/user.hpp \ - /home/pblunsom/packages/include/boost/config/select_compiler_config.hpp \ - /home/pblunsom/packages/include/boost/config/compiler/gcc.hpp \ - /home/pblunsom/packages/include/boost/config/select_stdlib_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/utility.hpp \ - /home/pblunsom/packages/include/boost/config/stdlib/libstdcpp3.hpp \ - /home/pblunsom/packages/include/boost/config/select_platform_config.hpp \ - /home/pblunsom/packages/include/boost/config/platform/linux.hpp \ - /home/pblunsom/packages/include/boost/config/posix_features.hpp \ - /home/pblunsom/packages/include/boost/config/suffix.hpp \ - /home/pblunsom/packages/include/boost/version.hpp \ - /home/pblunsom/packages/include/boost/config/auto_link.hpp \ - /home/pblunsom/packages/include/boost/program_options/option.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/cmdline.hpp \ - /home/pblunsom/packages/include/boost/program_options/errors.hpp \ - /home/pblunsom/packages/include/boost/program_options/cmdline.hpp \ - /home/pblunsom/packages/include/boost/program_options/options_description.hpp \ - /home/pblunsom/packages/include/boost/program_options/value_semantic.hpp \ - /home/pblunsom/packages/include/boost/any.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/broken_compiler_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_support.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/gcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/workaround.hpp \ - /home/pblunsom/packages/include/boost/detail/workaround.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ctps.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/template_arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/int.hpp \ - /home/pblunsom/packages/include/boost/mpl/int_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/adl_barrier.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/adl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/intel.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nttp_decl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/nttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/integral_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/static_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/static_cast.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/config.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/params.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bool.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/error.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/auto_rec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/eat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/inc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/inc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/overload_resolution.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/config.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/attribute_noreturn.hpp \ - /home/pblunsom/packages/include/boost/exception/exception.hpp \ - /home/pblunsom/packages/include/boost/current_function.hpp \ - /home/pblunsom/packages/include/boost/static_assert.hpp \ - /home/pblunsom/packages/include/boost/function/function1.hpp \ - /home/pblunsom/packages/include/boost/function/detail/maybe_include.hpp \ - /home/pblunsom/packages/include/boost/function/function_template.hpp \ - /home/pblunsom/packages/include/boost/function/detail/prologue.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/functional.hpp \ - /home/pblunsom/packages/include/boost/function/function_base.hpp \ - /home/pblunsom/packages/include/boost/detail/sp_typeinfo.hpp \ - /home/pblunsom/packages/include/boost/assert.hpp \ - /home/pblunsom/packages/include/boost/integer.hpp \ - /home/pblunsom/packages/include/boost/integer_fwd.hpp \ - /home/pblunsom/packages/include/boost/limits.hpp \ - /home/pblunsom/packages/include/boost/cstdint.hpp \ - /home/pblunsom/packages/include/boost/integer_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_copy.hpp \ - /home/pblunsom/packages/include/boost/type_traits/intrinsics.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_same.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/cv_traits_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pod.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_void.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_scalar.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_arithmetic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_integral.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_float.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_or.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_enum.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_function_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_mem_fun_pointer_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_and.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_not.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/composite_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_array.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_union.hpp \ - /home/pblunsom/packages/include/boost/type_traits/ice.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/yes_no_type.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_eq.hpp \ - /home/pblunsom/packages/include/boost/ref.hpp \ - /home/pblunsom/packages/include/boost/utility/addressof.hpp \ - /home/pblunsom/packages/include/boost/mpl/if.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/value_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/integral.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/eti.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/void_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_arity_param.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/dtp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/def_params_tail.hpp \ - /home/pblunsom/packages/include/boost/mpl/limits/arity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/and.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/add.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/dec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/adt.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/check.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/compl.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/detail/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/sub.hpp \ - /home/pblunsom/packages/include/boost/type_traits/alignment_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/utility/enable_if.hpp \ - /home/pblunsom/packages/include/boost/function_equal.hpp \ - /home/pblunsom/packages/include/boost/function/function_fwd.hpp \ - /home/pblunsom/packages/include/boost/mem_fn.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn.hpp \ - /home/pblunsom/packages/include/boost/get_pointer.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/memory.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn_template.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn_cc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/rem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/enum_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params.hpp \ - /home/pblunsom/packages/include/boost/detail/no_exceptions_support.hpp \ - /home/pblunsom/packages/include/boost/lexical_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/make_unsigned.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_signed.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_unsigned.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_volatile.hpp \ - /home/pblunsom/packages/include/boost/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/lcast_precision.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_abstract.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/value_semantic.hpp \ - /home/pblunsom/packages/include/boost/function.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/data.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/def.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/iter/forward1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/lower1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/shared.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/upper1.hpp \ - /home/pblunsom/packages/include/boost/function/detail/function_iterate.hpp \ - /home/pblunsom/packages/include/boost/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/checked_delete.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/shared_count.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/bad_weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_has_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_impl.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_convertible.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_pool.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/yield_k.hpp \ - /home/pblunsom/packages/include/boost/memory_order.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/operator_bool.hpp \ - /home/pblunsom/packages/include/boost/program_options/positional_options.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/parsers.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/convert.hpp \ - /home/pblunsom/packages/include/boost/program_options/variables_map.hpp \ - /home/pblunsom/packages/include/boost/scoped_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_ptr.hpp \ - /home/pblunsom/packages/include/boost/mpi/environment.hpp \ - /home/pblunsom/packages/include/boost/mpi/config.hpp \ - /home/pblunsom/packages/include/mpi.h \ - /home/pblunsom/packages/include/mpio.h \ - /home/pblunsom/packages/include/mpi.h \ - /home/pblunsom/packages/include/mpicxx.h \ - /home/pblunsom/packages/include/boost/noncopyable.hpp \ - /home/pblunsom/packages/include/boost/optional.hpp \ - /home/pblunsom/packages/include/boost/optional/optional.hpp \ - /home/pblunsom/packages/include/boost/type.hpp \ - /home/pblunsom/packages/include/boost/type_traits/type_with_alignment.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/detail/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/append.hpp \ - /home/pblunsom/packages/include/boost/mpl/not.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nested_type_wknd.hpp \ - /home/pblunsom/packages/include/boost/detail/reference_content.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_copy.hpp \ - /home/pblunsom/packages/include/boost/mpl/void.hpp \ - /home/pblunsom/packages/include/boost/none.hpp \ - /home/pblunsom/packages/include/boost/none_t.hpp \ - /home/pblunsom/packages/include/boost/utility/compare_pointees.hpp \ - /home/pblunsom/packages/include/boost/optional/optional_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpi/communicator.hpp \ - /home/pblunsom/packages/include/boost/mpi/exception.hpp \ - /home/pblunsom/packages/include/boost/mpi/datatype.hpp \ - /home/pblunsom/packages/include/boost/mpi/datatype_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/use_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/include_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/compiler.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/stringize.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/and.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/mpi_datatype_cache.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/mpi_datatype_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/oserializer.hpp \ - /home/pblunsom/packages/include/boost/mpl/eval_if.hpp \ - /home/pblunsom/packages/include/boost/mpl/equal_to.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/comparison_op.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/numeric_op.hpp \ - /home/pblunsom/packages/include/boost/mpl/numeric_cast.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/type_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/yes_no.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/arrays.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc_typename.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_never_true.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/numeric_cast_utils.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/forwarding.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_eti_base.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/is_msvc_eti_arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/equal_to.hpp \ - /home/pblunsom/packages/include/boost/mpl/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/mpl/identity.hpp \ - /home/pblunsom/packages/include/boost/serialization/extended_type_info_typeid.hpp \ - /home/pblunsom/packages/include/boost/serialization/static_warning.hpp \ - /home/pblunsom/packages/include/boost/mpl/print.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_polymorphic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_const.hpp \ - /home/pblunsom/packages/include/boost/serialization/singleton.hpp \ - /home/pblunsom/packages/include/boost/serialization/force_include.hpp \ - /home/pblunsom/packages/include/boost/serialization/extended_type_info.hpp \ - /home/pblunsom/packages/include/boost/serialization/config.hpp \ - /home/pblunsom/packages/include/boost/config/abi_prefix.hpp \ - /home/pblunsom/packages/include/boost/config/abi_suffix.hpp \ - /home/pblunsom/packages/include/boost/serialization/factory.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/not.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/not_equal.hpp \ - /home/pblunsom/packages/include/boost/serialization/access.hpp \ - /home/pblunsom/packages/include/boost/serialization/pfto.hpp \ - /home/pblunsom/packages/include/boost/serialization/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/serialization/smart_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_and_derived.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_pointer.hpp \ - /home/pblunsom/packages/include/boost/serialization/assume_abstract.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_extent.hpp \ - /home/pblunsom/packages/include/boost/serialization/serialization.hpp \ - /home/pblunsom/packages/include/boost/serialization/strong_typedef.hpp \ - /home/pblunsom/packages/include/boost/operators.hpp \ - /home/pblunsom/packages/include/boost/iterator.hpp \ - /home/pblunsom/packages/include/boost/serialization/nvp.hpp \ - /home/pblunsom/packages/include/boost/serialization/level.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_fundamental.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_class.hpp \ - /home/pblunsom/packages/include/boost/serialization/level_enum.hpp \ - /home/pblunsom/packages/include/boost/serialization/tracking.hpp \ - /home/pblunsom/packages/include/boost/mpl/greater.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/greater.hpp \ - /home/pblunsom/packages/include/boost/serialization/tracking_enum.hpp \ - /home/pblunsom/packages/include/boost/serialization/type_info_implementation.hpp \ - /home/pblunsom/packages/include/boost/serialization/traits.hpp \ - /home/pblunsom/packages/include/boost/serialization/split_member.hpp \ - /home/pblunsom/packages/include/boost/serialization/base_object.hpp \ - /home/pblunsom/packages/include/boost/serialization/void_cast_fwd.hpp \ - /home/pblunsom/packages/include/boost/serialization/wrapper.hpp \ - /home/pblunsom/packages/include/boost/serialization/version.hpp \ - /home/pblunsom/packages/include/boost/mpl/assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/pp_counter.hpp \ - /home/pblunsom/packages/include/boost/mpl/less.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/less.hpp \ - /home/pblunsom/packages/include/boost/mpl/comparison.hpp \ - /home/pblunsom/packages/include/boost/mpl/not_equal_to.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/not_equal_to.hpp \ - /home/pblunsom/packages/include/boost/mpl/less_equal.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/less_equal.hpp \ - /home/pblunsom/packages/include/boost/serialization/void_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_virtual_base_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_of.hpp \ - /home/pblunsom/packages/include/boost/serialization/array.hpp \ - /home/pblunsom/packages/include/boost/mpl/always.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arg_typedef.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/next.hpp \ - /home/pblunsom/packages/include/boost/mpl/next_prior.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/common_name_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/protect.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_type.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply.hpp \ - /home/pblunsom/packages/include/boost/array.hpp \ - /home/pblunsom/packages/include/boost/swap.hpp \ - /home/pblunsom/packages/include/boost/utility/swap.hpp \ - /home/pblunsom/packages/include/boost/detail/iterator.hpp \ - /home/pblunsom/packages/include/boost/serialization/collection_size_type.hpp \ - /home/pblunsom/packages/include/boost/archive/archive_exception.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/decl.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/abi_prefix.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/abi_suffix.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/basic_archive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/auto_link_archive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_oserializer.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_serializer.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_pointer_oserializer.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/archive_serializer_map.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/check.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/ignore_skeleton_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/common_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/interface_oarchive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/mpi_datatype_primitive.hpp \ - /home/pblunsom/packages/include/boost/serialization/detail/get_data.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/register_archive.hpp \ - /home/pblunsom/packages/include/boost/mpi/packed_oarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/basic_binary_oarchive.hpp \ - /home/pblunsom/packages/include/boost/serialization/string.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/packed_oprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/allocator.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/binary_buffer_oprimitive.hpp \ - /home/pblunsom/packages/include/boost/serialization/is_bitwise_serializable.hpp \ - /home/pblunsom/packages/include/boost/mpi/packed_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/basic_binary_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/common_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_pointer_iserializer.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/interface_iarchive.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/iserializer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_new_operator.hpp \ - /home/pblunsom/packages/include/boost/archive/detail/basic_iserializer.hpp \ - /home/pblunsom/packages/include/boost/archive/shared_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/serialization/shared_ptr_132.hpp \ - /home/pblunsom/packages/include/boost/serialization/split_free.hpp \ - /home/pblunsom/packages/include/boost/serialization/detail/shared_ptr_132.hpp \ - /home/pblunsom/packages/include/boost/serialization/detail/shared_count_132.hpp \ - /home/pblunsom/packages/include/boost/detail/lightweight_mutex.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/lightweight_mutex.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/lwm_pthreads.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/packed_iprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/binary_buffer_iprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/skeleton_and_content_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/point_to_point.hpp \ - /home/pblunsom/packages/include/boost/mpi/status.hpp \ - /home/pblunsom/packages/include/boost/mpi/request.hpp mpi-pyp-topics.hh \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_vector.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_sequence_adapter.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/reversible_ptr_container.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/scoped_deleter.hpp \ - /home/pblunsom/packages/include/boost/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/static_move_ptr.hpp \ - /home/pblunsom/packages/include/boost/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/detail/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_empty.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_reference.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/default_deleter.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_bounds.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/move.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/clone_allocator.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/nullable.hpp \ - /home/pblunsom/packages/include/boost/range/functions.hpp \ - /home/pblunsom/packages/include/boost/range/begin.hpp \ - /home/pblunsom/packages/include/boost/range/config.hpp \ - /home/pblunsom/packages/include/boost/range/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/mutable_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/detail/extract_optional_type.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_traits.hpp \ - /home/pblunsom/packages/include/boost/range/const_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/end.hpp \ - /home/pblunsom/packages/include/boost/range/detail/implementation_help.hpp \ - /home/pblunsom/packages/include/boost/range/detail/common.hpp \ - /home/pblunsom/packages/include/boost/range/detail/sfinae.hpp \ - /home/pblunsom/packages/include/boost/range/size.hpp \ - /home/pblunsom/packages/include/boost/range/difference_type.hpp \ - /home/pblunsom/packages/include/boost/range/distance.hpp \ - /home/pblunsom/packages/include/boost/range/empty.hpp \ - /home/pblunsom/packages/include/boost/range/rbegin.hpp \ - /home/pblunsom/packages/include/boost/range/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/utility.hpp \ - /home/pblunsom/packages/include/boost/utility/base_from_member.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat_from_to.hpp \ - /home/pblunsom/packages/include/boost/utility/binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/deduce_d.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mod.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/detail/div_base.hpp \ - /home/pblunsom/packages/include/boost/next_prior.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_adaptor.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_categories.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_def.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_undef.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_facade.hpp \ - /home/pblunsom/packages/include/boost/iterator/interoperable.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/facade_iterator_category.hpp \ - /home/pblunsom/packages/include/boost/detail/indirect_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_function.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/false_result.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_function_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/enable_if.hpp \ - /home/pblunsom/packages/include/boost/implicit_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_pointer.hpp \ - /home/pblunsom/packages/include/boost/range/rend.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/indirect_fun.hpp \ - /home/pblunsom/packages/include/boost/utility/result_of.hpp \ - /home/pblunsom/packages/include/boost/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/library.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/div.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mul.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_z.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/limits.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/assert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/line.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/apply.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_unary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/expand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/intercept.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/local.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/self.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/at.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitnor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitxor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/nor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/or.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/xor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_r.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_a_default.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_defaults.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/max.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/min.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/detail/split.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/subseq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_seq.hpp \ - /home/pblunsom/packages/include/boost/utility/detail/result_of_iterate.hpp \ - /home/pblunsom/packages/include/boost/pointee.hpp \ - /home/pblunsom/packages/include/boost/detail/is_incrementable.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/void_ptr_iterator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_real.hpp \ - /home/pblunsom/packages/include/boost/random/detail/config.hpp \ - /home/pblunsom/packages/include/boost/random/variate_generator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_01.hpp \ - /home/pblunsom/packages/include/boost/random/detail/pass_through_engine.hpp \ - /home/pblunsom/packages/include/boost/random/detail/ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/random/detail/disable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/enable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/uniform_int_float.hpp \ - /home/pblunsom/packages/include/boost/random/mersenne_twister.hpp \ - /home/pblunsom/packages/include/boost/random/linear_congruential.hpp \ - /home/pblunsom/packages/include/boost/random/detail/const_mod.hpp \ - /home/pblunsom/packages/include/boost/random/detail/seed.hpp \ - /home/pblunsom/packages/include/boost/random/inversive_congruential.hpp \ - /home/pblunsom/packages/include/boost/random/lagged_fibonacci.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/cmath.hpp mpi-pyp.hh \ - /home/pblunsom/packages/include/boost/tuple/tuple.hpp \ - /home/pblunsom/packages/include/boost/tuple/detail/tuple_basic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/cv_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/function_traits.hpp \ - /home/pblunsom/packages/include/boost/serialization/map.hpp \ - /home/pblunsom/packages/include/boost/serialization/utility.hpp \ - /home/pblunsom/packages/include/boost/serialization/collections_save_imp.hpp \ - /home/pblunsom/packages/include/boost/serialization/collections_load_imp.hpp \ - /home/pblunsom/packages/include/boost/serialization/detail/stack_constructor.hpp \ - /home/pblunsom/packages/include/boost/aligned_storage.hpp \ - /home/pblunsom/packages/include/boost/mpi.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/all_gather.hpp \ - /home/pblunsom/packages/include/boost/serialization/vector.hpp \ - /home/pblunsom/packages/include/boost/serialization/collection_traits.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/broadcast.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/gather.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/all_reduce.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/reduce.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/computation_tree.hpp \ - /home/pblunsom/packages/include/boost/mpi/operations.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/all_to_all.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/scatter.hpp \ - /home/pblunsom/packages/include/boost/mpi/collectives/scan.hpp \ - /home/pblunsom/packages/include/boost/mpi/graph_communicator.hpp \ - /home/pblunsom/packages/include/boost/graph/graph_traits.hpp \ - /home/pblunsom/packages/include/boost/pending/property.hpp \ - /home/pblunsom/packages/include/boost/pending/detail/property.hpp \ - /home/pblunsom/packages/include/boost/type_traits/same_traits.hpp \ - /home/pblunsom/packages/include/boost/graph/properties.hpp \ - /home/pblunsom/packages/include/boost/property_map/property_map.hpp \ - /home/pblunsom/packages/include/boost/pending/cstddef.hpp \ - /home/pblunsom/packages/include/boost/concept_check.hpp \ - /home/pblunsom/packages/include/boost/concept/assert.hpp \ - /home/pblunsom/packages/include/boost/concept/detail/general.hpp \ - /home/pblunsom/packages/include/boost/concept/detail/has_constraints.hpp \ - /home/pblunsom/packages/include/boost/type_traits/conversion_traits.hpp \ - /home/pblunsom/packages/include/boost/concept/usage.hpp \ - /home/pblunsom/packages/include/boost/concept/detail/concept_def.hpp \ - /home/pblunsom/packages/include/boost/concept/detail/concept_undef.hpp \ - /home/pblunsom/packages/include/boost/concept_archetype.hpp \ - /home/pblunsom/packages/include/boost/property_map/vector_property_map.hpp \ - /home/pblunsom/packages/include/boost/graph/property_maps/constant_property_map.hpp \ - /home/pblunsom/packages/include/boost/graph/property_maps/null_property_map.hpp \ - /home/pblunsom/packages/include/boost/iterator/counting_iterator.hpp \ - /home/pblunsom/packages/include/boost/detail/numeric_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_assign.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_assign.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_constructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_constructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_virtual_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_compound.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_floating_point.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_object_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_object.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_stateless.hpp \ - /home/pblunsom/packages/include/boost/type_traits/rank.hpp \ - /home/pblunsom/packages/include/boost/type_traits/extent.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_all_extents.hpp \ - /home/pblunsom/packages/include/boost/type_traits/aligned_storage.hpp \ - /home/pblunsom/packages/include/boost/type_traits/floating_point_promotion.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_promotion.hpp \ - /home/pblunsom/packages/include/boost/type_traits/promote.hpp \ - /home/pblunsom/packages/include/boost/type_traits/make_signed.hpp \ - /home/pblunsom/packages/include/boost/type_traits/decay.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_complex.hpp \ - /home/pblunsom/packages/include/boost/detail/select_type.hpp \ - /home/pblunsom/packages/include/boost/graph/iteration_macros.hpp \ - /home/pblunsom/packages/include/boost/shared_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_array.hpp \ - /home/pblunsom/packages/include/boost/mpi/group.hpp \ - /home/pblunsom/packages/include/boost/mpi/intercommunicator.hpp \ - /home/pblunsom/packages/include/boost/mpi/nonblocking.hpp \ - /home/pblunsom/packages/include/boost/mpi/skeleton_and_content.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/forward_skeleton_iarchive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/forward_skeleton_oarchive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/ignore_iprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/ignore_oprimitive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/content_oarchive.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/broadcast_sc.hpp \ - /home/pblunsom/packages/include/boost/mpi/detail/communicator_sc.hpp \ - /home/pblunsom/packages/include/boost/mpi/timer.hpp pyp.hh \ - slice-sampler.h log_add.h mt19937ar.h corpus.hh mpi-corpus.hh \ - contexts_corpus.hh contexts_lexer.h ../../../decoder/dict.h \ - /home/pblunsom/packages/include/boost/functional/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash_fwd.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/float_functions.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/limits.hpp \ - /home/pblunsom/packages/include/boost/integer/static_log2.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float_generic.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/extensions.hpp \ - /home/pblunsom/packages/include/boost/detail/container_fwd.hpp \ - ../../../decoder/wordid.h gzstream.hh -pyp-topics.o: pyp-topics.cc timing.h clock_gettime_stub.c pyp-topics.hh \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_vector.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_sequence_adapter.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/reversible_ptr_container.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/assert.hpp \ - /home/pblunsom/packages/include/boost/config.hpp \ - /home/pblunsom/packages/include/boost/config/user.hpp \ - /home/pblunsom/packages/include/boost/config/select_compiler_config.hpp \ - /home/pblunsom/packages/include/boost/config/compiler/gcc.hpp \ - /home/pblunsom/packages/include/boost/config/select_stdlib_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/utility.hpp \ - /home/pblunsom/packages/include/boost/config/stdlib/libstdcpp3.hpp \ - /home/pblunsom/packages/include/boost/config/select_platform_config.hpp \ - /home/pblunsom/packages/include/boost/config/platform/linux.hpp \ - /home/pblunsom/packages/include/boost/config/posix_features.hpp \ - /home/pblunsom/packages/include/boost/config/suffix.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/scoped_deleter.hpp \ - /home/pblunsom/packages/include/boost/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/checked_delete.hpp \ - /home/pblunsom/packages/include/boost/detail/workaround.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/operator_bool.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/static_move_ptr.hpp \ - /home/pblunsom/packages/include/boost/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/detail/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/broken_compiler_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_support.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/gcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/workaround.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ctps.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/cv_traits_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/template_arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/int.hpp \ - /home/pblunsom/packages/include/boost/mpl/int_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/adl_barrier.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/adl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/intel.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nttp_decl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/nttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/integral_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/static_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/static_cast.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/config.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/params.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bool.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/error.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/auto_rec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/eat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/inc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/inc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/overload_resolution.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_empty.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/type_traits/intrinsics.hpp \ - /home/pblunsom/packages/include/boost/type_traits/config.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_same.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/yes_no_type.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_array.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/ice.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_or.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_and.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_not.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_eq.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_arithmetic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_integral.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_float.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_void.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_abstract.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_class.hpp \ - /home/pblunsom/packages/include/boost/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/call_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_function_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_mem_fun_pointer_impl.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/default_deleter.hpp \ - /home/pblunsom/packages/include/boost/mpl/if.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/value_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/integral.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/eti.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/void_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_arity_param.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/dtp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/def_params_tail.hpp \ - /home/pblunsom/packages/include/boost/mpl/limits/arity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/and.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/add.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/dec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/adt.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/check.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/compl.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/detail/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/sub.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_bounds.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/mpl/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/use_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nested_type_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/include_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/compiler.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/stringize.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/identity.hpp \ - /home/pblunsom/packages/include/boost/utility/enable_if.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/move.hpp \ - /home/pblunsom/packages/include/boost/static_assert.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/clone_allocator.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/nullable.hpp \ - /home/pblunsom/packages/include/boost/mpl/eval_if.hpp \ - /home/pblunsom/packages/include/boost/range/functions.hpp \ - /home/pblunsom/packages/include/boost/range/begin.hpp \ - /home/pblunsom/packages/include/boost/range/config.hpp \ - /home/pblunsom/packages/include/boost/range/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/mutable_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/detail/extract_optional_type.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/const_iterator.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_const.hpp \ - /home/pblunsom/packages/include/boost/range/end.hpp \ - /home/pblunsom/packages/include/boost/range/detail/implementation_help.hpp \ - /home/pblunsom/packages/include/boost/range/detail/common.hpp \ - /home/pblunsom/packages/include/boost/range/detail/sfinae.hpp \ - /home/pblunsom/packages/include/boost/range/size.hpp \ - /home/pblunsom/packages/include/boost/range/difference_type.hpp \ - /home/pblunsom/packages/include/boost/range/distance.hpp \ - /home/pblunsom/packages/include/boost/range/empty.hpp \ - /home/pblunsom/packages/include/boost/range/rbegin.hpp \ - /home/pblunsom/packages/include/boost/range/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator.hpp \ - /home/pblunsom/packages/include/boost/utility.hpp \ - /home/pblunsom/packages/include/boost/utility/addressof.hpp \ - /home/pblunsom/packages/include/boost/utility/base_from_member.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/rem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat_from_to.hpp \ - /home/pblunsom/packages/include/boost/utility/binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/deduce_d.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mod.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/detail/div_base.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/not.hpp \ - /home/pblunsom/packages/include/boost/next_prior.hpp \ - /home/pblunsom/packages/include/boost/noncopyable.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_adaptor.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_categories.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/not.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/yes_no.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/arrays.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/pp_counter.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arg_typedef.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/placeholders.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_undef.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_facade.hpp \ - /home/pblunsom/packages/include/boost/iterator/interoperable.hpp \ - /home/pblunsom/packages/include/boost/mpl/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/or.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/facade_iterator_category.hpp \ - /home/pblunsom/packages/include/boost/detail/indirect_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_function.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/false_result.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_function_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_pointer.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/enable_if.hpp \ - /home/pblunsom/packages/include/boost/implicit_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pod.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_scalar.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/always.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/type_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc_typename.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_never_true.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/next.hpp \ - /home/pblunsom/packages/include/boost/mpl/next_prior.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/common_name_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/protect.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/void.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_type.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply.hpp \ - /home/pblunsom/packages/include/boost/range/rend.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/indirect_fun.hpp \ - /home/pblunsom/packages/include/boost/utility/result_of.hpp \ - /home/pblunsom/packages/include/boost/type.hpp \ - /home/pblunsom/packages/include/boost/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/library.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/div.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mul.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/data.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/not_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_z.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/limits.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/assert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/line.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/def.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/apply.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_unary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/expand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/intercept.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/local.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/self.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/append.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/at.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/detail/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitnor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitxor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/nor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/or.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/xor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_r.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_a_default.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_defaults.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/max.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/min.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/detail/split.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/subseq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/iter/forward1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/lower1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/shared.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/upper1.hpp \ - /home/pblunsom/packages/include/boost/utility/detail/result_of_iterate.hpp \ - /home/pblunsom/packages/include/boost/pointee.hpp \ - /home/pblunsom/packages/include/boost/detail/is_incrementable.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/void_ptr_iterator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_real.hpp \ - /home/pblunsom/packages/include/boost/limits.hpp \ - /home/pblunsom/packages/include/boost/random/detail/config.hpp \ - /home/pblunsom/packages/include/boost/random/variate_generator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_01.hpp \ - /home/pblunsom/packages/include/boost/random/detail/pass_through_engine.hpp \ - /home/pblunsom/packages/include/boost/random/detail/ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/random/detail/disable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/enable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/uniform_int_float.hpp \ - /home/pblunsom/packages/include/boost/random/mersenne_twister.hpp \ - /home/pblunsom/packages/include/boost/integer_traits.hpp \ - /home/pblunsom/packages/include/boost/cstdint.hpp \ - /home/pblunsom/packages/include/boost/random/linear_congruential.hpp \ - /home/pblunsom/packages/include/boost/random/detail/const_mod.hpp \ - /home/pblunsom/packages/include/boost/random/detail/seed.hpp pyp.hh \ - slice-sampler.h log_add.h mt19937ar.h corpus.hh \ - /home/pblunsom/packages/include/boost/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/memory.hpp \ - /home/pblunsom/packages/include/boost/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/attribute_noreturn.hpp \ - /home/pblunsom/packages/include/boost/exception/exception.hpp \ - /home/pblunsom/packages/include/boost/current_function.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/shared_count.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/bad_weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_has_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp \ - /home/pblunsom/packages/include/boost/detail/sp_typeinfo.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_impl.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_convertible.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_pool.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/yield_k.hpp \ - /home/pblunsom/packages/include/boost/memory_order.hpp workers.hh \ - /home/pblunsom/packages/include/boost/bind.hpp \ - /home/pblunsom/packages/include/boost/bind/bind.hpp \ - /home/pblunsom/packages/include/boost/ref.hpp \ - /home/pblunsom/packages/include/boost/mem_fn.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn.hpp \ - /home/pblunsom/packages/include/boost/get_pointer.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn_template.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn_cc.hpp \ - /home/pblunsom/packages/include/boost/is_placeholder.hpp \ - /home/pblunsom/packages/include/boost/bind/arg.hpp \ - /home/pblunsom/packages/include/boost/visit_each.hpp \ - /home/pblunsom/packages/include/boost/bind/storage.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_template.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_mf_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_mf2_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/placeholders.hpp \ - /home/pblunsom/packages/include/boost/function.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iterate.hpp \ - /home/pblunsom/packages/include/boost/function/detail/prologue.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/functional.hpp \ - /home/pblunsom/packages/include/boost/function/function_base.hpp \ - /home/pblunsom/packages/include/boost/integer.hpp \ - /home/pblunsom/packages/include/boost/integer_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_copy.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/composite_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_union.hpp \ - /home/pblunsom/packages/include/boost/type_traits/alignment_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/function_equal.hpp \ - /home/pblunsom/packages/include/boost/function/function_fwd.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/enum_params.hpp \ - /home/pblunsom/packages/include/boost/function/detail/function_iterate.hpp \ - /home/pblunsom/packages/include/boost/function/detail/maybe_include.hpp \ - /home/pblunsom/packages/include/boost/function/function_template.hpp \ - /home/pblunsom/packages/include/boost/detail/no_exceptions_support.hpp \ - /home/pblunsom/packages/include/boost/thread/thread.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/platform.hpp \ - /home/pblunsom/packages/include/boost/config/requires_threads.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_data.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/config.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/platform.hpp \ - /home/pblunsom/packages/include/boost/config/auto_link.hpp \ - /home/pblunsom/packages/include/boost/thread/exceptions.hpp \ - /home/pblunsom/packages/include/boost/config/abi_prefix.hpp \ - /home/pblunsom/packages/include/boost/config/abi_suffix.hpp \ - /home/pblunsom/packages/include/boost/enable_shared_from_this.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/enable_shared_from_this.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/thread/mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/locks.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/move.hpp \ - /home/pblunsom/packages/include/boost/thread/thread_time.hpp \ - /home/pblunsom/packages/include/boost/date_time/microsec_time_clock.hpp \ - /home/pblunsom/packages/include/boost/date_time/compiler_config.hpp \ - /home/pblunsom/packages/include/boost/date_time/locale_config.hpp \ - /home/pblunsom/packages/include/boost/date_time/c_time.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_clock.hpp \ - /home/pblunsom/packages/include/boost/date_time/filetime_functions.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/ptime.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_system.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/cmath.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_duration.hpp \ - /home/pblunsom/packages/include/boost/operators.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/special_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_resolution_traits.hpp \ - /home/pblunsom/packages/include/boost/date_time/int_adapter.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/gregorian_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/date.hpp \ - /home/pblunsom/packages/include/boost/date_time/year_month_day.hpp \ - /home/pblunsom/packages/include/boost/date_time/period.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_calendar.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_weekday.hpp \ - /home/pblunsom/packages/include/boost/date_time/constrained_value.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_and_derived.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_day_of_year.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian_calendar.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian_calendar.ipp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_ymd.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_day.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_year.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_month.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_duration_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_duration_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_date.hpp \ - /home/pblunsom/packages/include/boost/date_time/adjust_functors.hpp \ - /home/pblunsom/packages/include/boost/date_time/wrapping_int.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_generators.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_clock_device.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_iterator.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_system_split.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_system_counted.hpp \ - /home/pblunsom/packages/include/boost/date_time/time.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/date_duration_operators.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/time_period.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_iterator.hpp \ - /home/pblunsom/packages/include/boost/date_time/dst_rules.hpp \ - /home/pblunsom/packages/include/boost/thread/xtime.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/conversion.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/conversion.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/timespec.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/pthread_mutex_scoped_lock.hpp \ - /home/pblunsom/packages/include/boost/optional.hpp \ - /home/pblunsom/packages/include/boost/optional/optional.hpp \ - /home/pblunsom/packages/include/boost/type_traits/type_with_alignment.hpp \ - /home/pblunsom/packages/include/boost/detail/reference_content.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_copy.hpp \ - /home/pblunsom/packages/include/boost/none.hpp \ - /home/pblunsom/packages/include/boost/none_t.hpp \ - /home/pblunsom/packages/include/boost/utility/compare_pointees.hpp \ - /home/pblunsom/packages/include/boost/optional/optional_fwd.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/condition_variable_fwd.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_heap_alloc.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_heap_alloc.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_interruption.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_group.hpp \ - /home/pblunsom/packages/include/boost/thread/shared_mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/shared_mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/condition_variable.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/condition_variable.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_data.hpp \ - /home/pblunsom/packages/include/boost/thread/future.hpp \ - /home/pblunsom/packages/include/boost/exception_ptr.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/exception_ptr.hpp \ - /home/pblunsom/packages/include/boost/scoped_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_ptr.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_fundamental.hpp \ - /home/pblunsom/packages/include/boost/thread/condition.hpp -train-contexts.o: train-contexts.cc \ - /home/pblunsom/packages/include/boost/program_options/parsers.hpp \ - /home/pblunsom/packages/include/boost/program_options/config.hpp \ - /home/pblunsom/packages/include/boost/config.hpp \ - /home/pblunsom/packages/include/boost/config/user.hpp \ - /home/pblunsom/packages/include/boost/config/select_compiler_config.hpp \ - /home/pblunsom/packages/include/boost/config/compiler/gcc.hpp \ - /home/pblunsom/packages/include/boost/config/select_stdlib_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/utility.hpp \ - /home/pblunsom/packages/include/boost/config/stdlib/libstdcpp3.hpp \ - /home/pblunsom/packages/include/boost/config/select_platform_config.hpp \ - /home/pblunsom/packages/include/boost/config/platform/linux.hpp \ - /home/pblunsom/packages/include/boost/config/posix_features.hpp \ - /home/pblunsom/packages/include/boost/config/suffix.hpp \ - /home/pblunsom/packages/include/boost/version.hpp \ - /home/pblunsom/packages/include/boost/config/auto_link.hpp \ - /home/pblunsom/packages/include/boost/program_options/option.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/cmdline.hpp \ - /home/pblunsom/packages/include/boost/program_options/errors.hpp \ - /home/pblunsom/packages/include/boost/program_options/cmdline.hpp \ - /home/pblunsom/packages/include/boost/program_options/options_description.hpp \ - /home/pblunsom/packages/include/boost/program_options/value_semantic.hpp \ - /home/pblunsom/packages/include/boost/any.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/broken_compiler_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_support.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/gcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/workaround.hpp \ - /home/pblunsom/packages/include/boost/detail/workaround.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ctps.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/template_arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/int.hpp \ - /home/pblunsom/packages/include/boost/mpl/int_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/adl_barrier.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/adl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/intel.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nttp_decl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/nttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/integral_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/static_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/static_cast.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/config.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/params.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bool.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/error.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/auto_rec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/eat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/inc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/inc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/overload_resolution.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/config.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/attribute_noreturn.hpp \ - /home/pblunsom/packages/include/boost/exception/exception.hpp \ - /home/pblunsom/packages/include/boost/current_function.hpp \ - /home/pblunsom/packages/include/boost/static_assert.hpp \ - /home/pblunsom/packages/include/boost/function/function1.hpp \ - /home/pblunsom/packages/include/boost/function/detail/maybe_include.hpp \ - /home/pblunsom/packages/include/boost/function/function_template.hpp \ - /home/pblunsom/packages/include/boost/function/detail/prologue.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/functional.hpp \ - /home/pblunsom/packages/include/boost/function/function_base.hpp \ - /home/pblunsom/packages/include/boost/detail/sp_typeinfo.hpp \ - /home/pblunsom/packages/include/boost/assert.hpp \ - /home/pblunsom/packages/include/boost/integer.hpp \ - /home/pblunsom/packages/include/boost/integer_fwd.hpp \ - /home/pblunsom/packages/include/boost/limits.hpp \ - /home/pblunsom/packages/include/boost/cstdint.hpp \ - /home/pblunsom/packages/include/boost/integer_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_copy.hpp \ - /home/pblunsom/packages/include/boost/type_traits/intrinsics.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_same.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/cv_traits_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pod.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_void.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_scalar.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_arithmetic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_integral.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_float.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_or.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_enum.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_function_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_mem_fun_pointer_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_and.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_not.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/composite_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_array.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_union.hpp \ - /home/pblunsom/packages/include/boost/type_traits/ice.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/yes_no_type.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_eq.hpp \ - /home/pblunsom/packages/include/boost/ref.hpp \ - /home/pblunsom/packages/include/boost/utility/addressof.hpp \ - /home/pblunsom/packages/include/boost/mpl/if.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/value_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/integral.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/eti.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/void_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_arity_param.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/dtp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/def_params_tail.hpp \ - /home/pblunsom/packages/include/boost/mpl/limits/arity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/and.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/add.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/dec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/adt.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/check.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/compl.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/detail/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/sub.hpp \ - /home/pblunsom/packages/include/boost/type_traits/alignment_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/utility/enable_if.hpp \ - /home/pblunsom/packages/include/boost/function_equal.hpp \ - /home/pblunsom/packages/include/boost/function/function_fwd.hpp \ - /home/pblunsom/packages/include/boost/mem_fn.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn.hpp \ - /home/pblunsom/packages/include/boost/get_pointer.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/memory.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn_template.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn_cc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/rem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/enum_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params.hpp \ - /home/pblunsom/packages/include/boost/detail/no_exceptions_support.hpp \ - /home/pblunsom/packages/include/boost/lexical_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/make_unsigned.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_signed.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_unsigned.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_volatile.hpp \ - /home/pblunsom/packages/include/boost/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/lcast_precision.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_abstract.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/value_semantic.hpp \ - /home/pblunsom/packages/include/boost/function.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/data.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/def.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/iter/forward1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/lower1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/shared.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/upper1.hpp \ - /home/pblunsom/packages/include/boost/function/detail/function_iterate.hpp \ - /home/pblunsom/packages/include/boost/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/checked_delete.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/shared_count.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/bad_weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_has_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_impl.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_convertible.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_pool.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/yield_k.hpp \ - /home/pblunsom/packages/include/boost/memory_order.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/operator_bool.hpp \ - /home/pblunsom/packages/include/boost/program_options/positional_options.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/parsers.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/convert.hpp \ - /home/pblunsom/packages/include/boost/program_options/variables_map.hpp \ - /home/pblunsom/packages/include/boost/scoped_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_ptr.hpp \ - pyp-topics.hh \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_vector.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_sequence_adapter.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/reversible_ptr_container.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/scoped_deleter.hpp \ - /home/pblunsom/packages/include/boost/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/static_move_ptr.hpp \ - /home/pblunsom/packages/include/boost/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/detail/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_empty.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_class.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/default_deleter.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_bounds.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/mpl/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/use_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nested_type_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/include_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/compiler.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/stringize.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/identity.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/move.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/clone_allocator.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/nullable.hpp \ - /home/pblunsom/packages/include/boost/mpl/eval_if.hpp \ - /home/pblunsom/packages/include/boost/range/functions.hpp \ - /home/pblunsom/packages/include/boost/range/begin.hpp \ - /home/pblunsom/packages/include/boost/range/config.hpp \ - /home/pblunsom/packages/include/boost/range/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/mutable_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/detail/extract_optional_type.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/const_iterator.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_const.hpp \ - /home/pblunsom/packages/include/boost/range/end.hpp \ - /home/pblunsom/packages/include/boost/range/detail/implementation_help.hpp \ - /home/pblunsom/packages/include/boost/range/detail/common.hpp \ - /home/pblunsom/packages/include/boost/range/detail/sfinae.hpp \ - /home/pblunsom/packages/include/boost/range/size.hpp \ - /home/pblunsom/packages/include/boost/range/difference_type.hpp \ - /home/pblunsom/packages/include/boost/range/distance.hpp \ - /home/pblunsom/packages/include/boost/range/empty.hpp \ - /home/pblunsom/packages/include/boost/range/rbegin.hpp \ - /home/pblunsom/packages/include/boost/range/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator.hpp \ - /home/pblunsom/packages/include/boost/utility.hpp \ - /home/pblunsom/packages/include/boost/utility/base_from_member.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat_from_to.hpp \ - /home/pblunsom/packages/include/boost/utility/binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/deduce_d.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mod.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/detail/div_base.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/not.hpp \ - /home/pblunsom/packages/include/boost/next_prior.hpp \ - /home/pblunsom/packages/include/boost/noncopyable.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_adaptor.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_categories.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/not.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/yes_no.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/arrays.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/pp_counter.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arg_typedef.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/placeholders.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_undef.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_facade.hpp \ - /home/pblunsom/packages/include/boost/iterator/interoperable.hpp \ - /home/pblunsom/packages/include/boost/mpl/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/or.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/facade_iterator_category.hpp \ - /home/pblunsom/packages/include/boost/detail/indirect_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_function.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/false_result.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_function_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_pointer.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/enable_if.hpp \ - /home/pblunsom/packages/include/boost/implicit_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_pointer.hpp \ - /home/pblunsom/packages/include/boost/mpl/always.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/type_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc_typename.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_never_true.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/next.hpp \ - /home/pblunsom/packages/include/boost/mpl/next_prior.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/common_name_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/protect.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/void.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_type.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply.hpp \ - /home/pblunsom/packages/include/boost/range/rend.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/indirect_fun.hpp \ - /home/pblunsom/packages/include/boost/utility/result_of.hpp \ - /home/pblunsom/packages/include/boost/type.hpp \ - /home/pblunsom/packages/include/boost/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/library.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/div.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mul.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/not_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_z.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/limits.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/assert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/line.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/apply.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_unary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/expand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/intercept.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/local.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/self.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/append.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/at.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/detail/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitnor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitxor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/nor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/or.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/xor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_r.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_a_default.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_defaults.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/max.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/min.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/detail/split.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/subseq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_seq.hpp \ - /home/pblunsom/packages/include/boost/utility/detail/result_of_iterate.hpp \ - /home/pblunsom/packages/include/boost/pointee.hpp \ - /home/pblunsom/packages/include/boost/detail/is_incrementable.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/void_ptr_iterator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_real.hpp \ - /home/pblunsom/packages/include/boost/random/detail/config.hpp \ - /home/pblunsom/packages/include/boost/random/variate_generator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_01.hpp \ - /home/pblunsom/packages/include/boost/random/detail/pass_through_engine.hpp \ - /home/pblunsom/packages/include/boost/random/detail/ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/random/detail/disable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/enable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/uniform_int_float.hpp \ - /home/pblunsom/packages/include/boost/random/mersenne_twister.hpp \ - /home/pblunsom/packages/include/boost/random/linear_congruential.hpp \ - /home/pblunsom/packages/include/boost/random/detail/const_mod.hpp \ - /home/pblunsom/packages/include/boost/random/detail/seed.hpp pyp.hh \ - slice-sampler.h log_add.h mt19937ar.h corpus.hh workers.hh \ - /home/pblunsom/packages/include/boost/bind.hpp \ - /home/pblunsom/packages/include/boost/bind/bind.hpp \ - /home/pblunsom/packages/include/boost/is_placeholder.hpp \ - /home/pblunsom/packages/include/boost/bind/arg.hpp \ - /home/pblunsom/packages/include/boost/visit_each.hpp \ - /home/pblunsom/packages/include/boost/bind/storage.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_template.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_mf_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_mf2_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/placeholders.hpp \ - /home/pblunsom/packages/include/boost/thread/thread.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/platform.hpp \ - /home/pblunsom/packages/include/boost/config/requires_threads.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_data.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/config.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/platform.hpp \ - /home/pblunsom/packages/include/boost/thread/exceptions.hpp \ - /home/pblunsom/packages/include/boost/config/abi_prefix.hpp \ - /home/pblunsom/packages/include/boost/config/abi_suffix.hpp \ - /home/pblunsom/packages/include/boost/enable_shared_from_this.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/enable_shared_from_this.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/thread/mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/locks.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/move.hpp \ - /home/pblunsom/packages/include/boost/thread/thread_time.hpp \ - /home/pblunsom/packages/include/boost/date_time/microsec_time_clock.hpp \ - /home/pblunsom/packages/include/boost/date_time/compiler_config.hpp \ - /home/pblunsom/packages/include/boost/date_time/locale_config.hpp \ - /home/pblunsom/packages/include/boost/date_time/c_time.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_clock.hpp \ - /home/pblunsom/packages/include/boost/date_time/filetime_functions.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/ptime.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_system.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/cmath.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_duration.hpp \ - /home/pblunsom/packages/include/boost/operators.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/special_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_resolution_traits.hpp \ - /home/pblunsom/packages/include/boost/date_time/int_adapter.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/gregorian_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/date.hpp \ - /home/pblunsom/packages/include/boost/date_time/year_month_day.hpp \ - /home/pblunsom/packages/include/boost/date_time/period.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_calendar.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_weekday.hpp \ - /home/pblunsom/packages/include/boost/date_time/constrained_value.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_and_derived.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_day_of_year.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian_calendar.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian_calendar.ipp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_ymd.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_day.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_year.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_month.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_duration_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_duration_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_date.hpp \ - /home/pblunsom/packages/include/boost/date_time/adjust_functors.hpp \ - /home/pblunsom/packages/include/boost/date_time/wrapping_int.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_generators.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_clock_device.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_iterator.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_system_split.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_system_counted.hpp \ - /home/pblunsom/packages/include/boost/date_time/time.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/date_duration_operators.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/time_period.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_iterator.hpp \ - /home/pblunsom/packages/include/boost/date_time/dst_rules.hpp \ - /home/pblunsom/packages/include/boost/thread/xtime.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/conversion.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/conversion.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/timespec.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/pthread_mutex_scoped_lock.hpp \ - /home/pblunsom/packages/include/boost/optional.hpp \ - /home/pblunsom/packages/include/boost/optional/optional.hpp \ - /home/pblunsom/packages/include/boost/type_traits/type_with_alignment.hpp \ - /home/pblunsom/packages/include/boost/detail/reference_content.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_copy.hpp \ - /home/pblunsom/packages/include/boost/none.hpp \ - /home/pblunsom/packages/include/boost/none_t.hpp \ - /home/pblunsom/packages/include/boost/utility/compare_pointees.hpp \ - /home/pblunsom/packages/include/boost/optional/optional_fwd.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/condition_variable_fwd.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_heap_alloc.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_heap_alloc.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_interruption.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_group.hpp \ - /home/pblunsom/packages/include/boost/thread/shared_mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/shared_mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/condition_variable.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/condition_variable.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_data.hpp \ - /home/pblunsom/packages/include/boost/thread/future.hpp \ - /home/pblunsom/packages/include/boost/exception_ptr.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/exception_ptr.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_fundamental.hpp \ - /home/pblunsom/packages/include/boost/thread/condition.hpp timing.h \ - clock_gettime_stub.c contexts_corpus.hh contexts_lexer.h \ - ../../../decoder/dict.h \ - /home/pblunsom/packages/include/boost/functional/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash_fwd.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/float_functions.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/limits.hpp \ - /home/pblunsom/packages/include/boost/integer/static_log2.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float_generic.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/extensions.hpp \ - /home/pblunsom/packages/include/boost/detail/container_fwd.hpp \ - ../../../decoder/wordid.h gzstream.hh -train.o: train.cc \ - /home/pblunsom/packages/include/boost/program_options/parsers.hpp \ - /home/pblunsom/packages/include/boost/program_options/config.hpp \ - /home/pblunsom/packages/include/boost/config.hpp \ - /home/pblunsom/packages/include/boost/config/user.hpp \ - /home/pblunsom/packages/include/boost/config/select_compiler_config.hpp \ - /home/pblunsom/packages/include/boost/config/compiler/gcc.hpp \ - /home/pblunsom/packages/include/boost/config/select_stdlib_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/utility.hpp \ - /home/pblunsom/packages/include/boost/config/stdlib/libstdcpp3.hpp \ - /home/pblunsom/packages/include/boost/config/select_platform_config.hpp \ - /home/pblunsom/packages/include/boost/config/platform/linux.hpp \ - /home/pblunsom/packages/include/boost/config/posix_features.hpp \ - /home/pblunsom/packages/include/boost/config/suffix.hpp \ - /home/pblunsom/packages/include/boost/version.hpp \ - /home/pblunsom/packages/include/boost/config/auto_link.hpp \ - /home/pblunsom/packages/include/boost/program_options/option.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/cmdline.hpp \ - /home/pblunsom/packages/include/boost/program_options/errors.hpp \ - /home/pblunsom/packages/include/boost/program_options/cmdline.hpp \ - /home/pblunsom/packages/include/boost/program_options/options_description.hpp \ - /home/pblunsom/packages/include/boost/program_options/value_semantic.hpp \ - /home/pblunsom/packages/include/boost/any.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/broken_compiler_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_support.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/gcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/workaround.hpp \ - /home/pblunsom/packages/include/boost/detail/workaround.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/ctps.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/template_arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/int.hpp \ - /home/pblunsom/packages/include/boost/mpl/int_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/adl_barrier.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/adl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/intel.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nttp_decl.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/nttp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/integral_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_tag.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/static_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/static_cast.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/config.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/params.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bool.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/comma.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/error.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/auto_rec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/eat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/inc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/inc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/overload_resolution.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/type_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/config.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_def.hpp \ - /home/pblunsom/packages/include/boost/type_traits/integral_constant.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool.hpp \ - /home/pblunsom/packages/include/boost/mpl/bool_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c.hpp \ - /home/pblunsom/packages/include/boost/mpl/integral_c_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/bool_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/attribute_noreturn.hpp \ - /home/pblunsom/packages/include/boost/exception/exception.hpp \ - /home/pblunsom/packages/include/boost/current_function.hpp \ - /home/pblunsom/packages/include/boost/static_assert.hpp \ - /home/pblunsom/packages/include/boost/function/function1.hpp \ - /home/pblunsom/packages/include/boost/function/detail/maybe_include.hpp \ - /home/pblunsom/packages/include/boost/function/function_template.hpp \ - /home/pblunsom/packages/include/boost/function/detail/prologue.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/functional.hpp \ - /home/pblunsom/packages/include/boost/function/function_base.hpp \ - /home/pblunsom/packages/include/boost/detail/sp_typeinfo.hpp \ - /home/pblunsom/packages/include/boost/assert.hpp \ - /home/pblunsom/packages/include/boost/integer.hpp \ - /home/pblunsom/packages/include/boost/integer_fwd.hpp \ - /home/pblunsom/packages/include/boost/limits.hpp \ - /home/pblunsom/packages/include/boost/cstdint.hpp \ - /home/pblunsom/packages/include/boost/integer_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_copy.hpp \ - /home/pblunsom/packages/include/boost/type_traits/intrinsics.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_same.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_volatile.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/cv_traits_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pod.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_void.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_scalar.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_arithmetic.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_integral.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_float.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_or.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_enum.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_member_function_pointer.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_mem_fun_pointer_impl.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_cv.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_and.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_not.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_trivial_destructor.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/composite_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_array.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_union.hpp \ - /home/pblunsom/packages/include/boost/type_traits/ice.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/yes_no_type.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/ice_eq.hpp \ - /home/pblunsom/packages/include/boost/ref.hpp \ - /home/pblunsom/packages/include/boost/utility/addressof.hpp \ - /home/pblunsom/packages/include/boost/mpl/if.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/value_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/integral.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/eti.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/void_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/lambda_arity_param.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/dtp.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessor/def_params_tail.hpp \ - /home/pblunsom/packages/include/boost/mpl/limits/arity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/and.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/identity.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/empty.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/add.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/dec.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_iif.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/adt.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/check.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/compl.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/detail/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/detail/while.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/sub.hpp \ - /home/pblunsom/packages/include/boost/type_traits/alignment_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t.hpp \ - /home/pblunsom/packages/include/boost/mpl/size_t_fwd.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/size_t_trait_undef.hpp \ - /home/pblunsom/packages/include/boost/utility/enable_if.hpp \ - /home/pblunsom/packages/include/boost/function_equal.hpp \ - /home/pblunsom/packages/include/boost/function/function_fwd.hpp \ - /home/pblunsom/packages/include/boost/mem_fn.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn.hpp \ - /home/pblunsom/packages/include/boost/get_pointer.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/memory.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn_template.hpp \ - /home/pblunsom/packages/include/boost/bind/mem_fn_cc.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/rem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/enum_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params.hpp \ - /home/pblunsom/packages/include/boost/detail/no_exceptions_support.hpp \ - /home/pblunsom/packages/include/boost/lexical_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/make_unsigned.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_signed.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_unsigned.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_const.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_volatile.hpp \ - /home/pblunsom/packages/include/boost/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/call_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/lcast_precision.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_abstract.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/value_semantic.hpp \ - /home/pblunsom/packages/include/boost/function.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/iterate.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/data.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/def.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/iter/forward1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/lower1.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot/detail/shared.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/detail/bounds/upper1.hpp \ - /home/pblunsom/packages/include/boost/function/detail/function_iterate.hpp \ - /home/pblunsom/packages/include/boost/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/shared_ptr.hpp \ - /home/pblunsom/packages/include/boost/checked_delete.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/shared_count.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/bad_weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_has_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_base_gcc_x86.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_counted_impl.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/sp_convertible.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_pool.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/spinlock_sync.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/yield_k.hpp \ - /home/pblunsom/packages/include/boost/memory_order.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/detail/operator_bool.hpp \ - /home/pblunsom/packages/include/boost/program_options/positional_options.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/parsers.hpp \ - /home/pblunsom/packages/include/boost/program_options/detail/convert.hpp \ - /home/pblunsom/packages/include/boost/program_options/variables_map.hpp \ - /home/pblunsom/packages/include/boost/scoped_ptr.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_ptr.hpp \ - pyp-topics.hh \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_vector.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/ptr_sequence_adapter.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/reversible_ptr_container.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/throw_exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/scoped_deleter.hpp \ - /home/pblunsom/packages/include/boost/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/scoped_array.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/static_move_ptr.hpp \ - /home/pblunsom/packages/include/boost/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/detail/compressed_pair.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_empty.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_reference.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_class.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/default_deleter.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_bounds.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/is_convertible.hpp \ - /home/pblunsom/packages/include/boost/mpl/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/use_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/nested_type_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/include_preprocessed.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/compiler.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/stringize.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/and.hpp \ - /home/pblunsom/packages/include/boost/mpl/identity.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/move.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/exception.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/clone_allocator.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/nullable.hpp \ - /home/pblunsom/packages/include/boost/mpl/eval_if.hpp \ - /home/pblunsom/packages/include/boost/range/functions.hpp \ - /home/pblunsom/packages/include/boost/range/begin.hpp \ - /home/pblunsom/packages/include/boost/range/config.hpp \ - /home/pblunsom/packages/include/boost/range/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/mutable_iterator.hpp \ - /home/pblunsom/packages/include/boost/range/detail/extract_optional_type.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_traits.hpp \ - /home/pblunsom/packages/include/boost/detail/iterator.hpp \ - /home/pblunsom/packages/include/boost/range/const_iterator.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_const.hpp \ - /home/pblunsom/packages/include/boost/range/end.hpp \ - /home/pblunsom/packages/include/boost/range/detail/implementation_help.hpp \ - /home/pblunsom/packages/include/boost/range/detail/common.hpp \ - /home/pblunsom/packages/include/boost/range/detail/sfinae.hpp \ - /home/pblunsom/packages/include/boost/range/size.hpp \ - /home/pblunsom/packages/include/boost/range/difference_type.hpp \ - /home/pblunsom/packages/include/boost/range/distance.hpp \ - /home/pblunsom/packages/include/boost/range/empty.hpp \ - /home/pblunsom/packages/include/boost/range/rbegin.hpp \ - /home/pblunsom/packages/include/boost/range/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator/reverse_iterator.hpp \ - /home/pblunsom/packages/include/boost/iterator.hpp \ - /home/pblunsom/packages/include/boost/utility.hpp \ - /home/pblunsom/packages/include/boost/utility/base_from_member.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/repeat_from_to.hpp \ - /home/pblunsom/packages/include/boost/utility/binary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/deduce_d.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_left.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/elem.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mod.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/detail/div_base.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/not.hpp \ - /home/pblunsom/packages/include/boost/next_prior.hpp \ - /home/pblunsom/packages/include/boost/noncopyable.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_adaptor.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_categories.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_def.hpp \ - /home/pblunsom/packages/include/boost/mpl/placeholders.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/arg_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/na_assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/assert.hpp \ - /home/pblunsom/packages/include/boost/mpl/not.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/yes_no.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/arrays.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/pp_counter.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arity_spec.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/arg_typedef.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/arg.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/placeholders.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/config_undef.hpp \ - /home/pblunsom/packages/include/boost/iterator/iterator_facade.hpp \ - /home/pblunsom/packages/include/boost/iterator/interoperable.hpp \ - /home/pblunsom/packages/include/boost/mpl/or.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/or.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/facade_iterator_category.hpp \ - /home/pblunsom/packages/include/boost/detail/indirect_traits.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_function.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/false_result.hpp \ - /home/pblunsom/packages/include/boost/type_traits/detail/is_function_ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/type_traits/remove_pointer.hpp \ - /home/pblunsom/packages/include/boost/iterator/detail/enable_if.hpp \ - /home/pblunsom/packages/include/boost/implicit_cast.hpp \ - /home/pblunsom/packages/include/boost/type_traits/add_pointer.hpp \ - /home/pblunsom/packages/include/boost/mpl/always.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/type_wrapper.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_xxx.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/msvc_typename.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/has_apply.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/msvc_never_true.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply_wrap.hpp \ - /home/pblunsom/packages/include/boost/mpl/lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind_fwd.hpp \ - /home/pblunsom/packages/include/boost/mpl/next.hpp \ - /home/pblunsom/packages/include/boost/mpl/next_prior.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/common_name_wknd.hpp \ - /home/pblunsom/packages/include/boost/mpl/protect.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/bind.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/void.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/has_type.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/config/bcc.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/quote.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/template_arity.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/full_lambda.hpp \ - /home/pblunsom/packages/include/boost/mpl/aux_/preprocessed/gcc/apply.hpp \ - /home/pblunsom/packages/include/boost/range/rend.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/indirect_fun.hpp \ - /home/pblunsom/packages/include/boost/utility/result_of.hpp \ - /home/pblunsom/packages/include/boost/type.hpp \ - /home/pblunsom/packages/include/boost/preprocessor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/library.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/div.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/arithmetic/mul.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/not_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_z.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/array/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/less.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/comparison/greater_equal.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/config/limits.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/control/expr_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/assert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/debug/line.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/apply.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/detail/is_unary.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/expand.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/facilities/intercept.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/local.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/iteration/self.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/append.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/at.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/cat.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/detail/for.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_list.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/size.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/list/transform.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitnor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/bitxor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/nor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/or.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/logical/xor.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/punctuation/paren_if.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/deduce_r.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_a_default.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_params_with_defaults.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_shifted_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_binary_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/repetition/enum_trailing_params.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/max.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/selection/min.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/enum.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/filter.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/first_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/detail/split.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/fold_right.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/reverse.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_i.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/for_each_product.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/insert.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/rest_n.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/pop_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_back.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/push_front.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/remove.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/replace.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/subseq.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_array.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/seq/to_tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/slot.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple.hpp \ - /home/pblunsom/packages/include/boost/preprocessor/tuple/to_seq.hpp \ - /home/pblunsom/packages/include/boost/utility/detail/result_of_iterate.hpp \ - /home/pblunsom/packages/include/boost/pointee.hpp \ - /home/pblunsom/packages/include/boost/detail/is_incrementable.hpp \ - /home/pblunsom/packages/include/boost/ptr_container/detail/void_ptr_iterator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_real.hpp \ - /home/pblunsom/packages/include/boost/random/detail/config.hpp \ - /home/pblunsom/packages/include/boost/random/variate_generator.hpp \ - /home/pblunsom/packages/include/boost/random/uniform_01.hpp \ - /home/pblunsom/packages/include/boost/random/detail/pass_through_engine.hpp \ - /home/pblunsom/packages/include/boost/random/detail/ptr_helper.hpp \ - /home/pblunsom/packages/include/boost/random/detail/disable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/enable_warnings.hpp \ - /home/pblunsom/packages/include/boost/random/detail/uniform_int_float.hpp \ - /home/pblunsom/packages/include/boost/random/mersenne_twister.hpp \ - /home/pblunsom/packages/include/boost/random/linear_congruential.hpp \ - /home/pblunsom/packages/include/boost/random/detail/const_mod.hpp \ - /home/pblunsom/packages/include/boost/random/detail/seed.hpp pyp.hh \ - slice-sampler.h log_add.h mt19937ar.h corpus.hh workers.hh \ - /home/pblunsom/packages/include/boost/bind.hpp \ - /home/pblunsom/packages/include/boost/bind/bind.hpp \ - /home/pblunsom/packages/include/boost/is_placeholder.hpp \ - /home/pblunsom/packages/include/boost/bind/arg.hpp \ - /home/pblunsom/packages/include/boost/visit_each.hpp \ - /home/pblunsom/packages/include/boost/bind/storage.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_template.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_mf_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/bind_mf2_cc.hpp \ - /home/pblunsom/packages/include/boost/bind/placeholders.hpp \ - /home/pblunsom/packages/include/boost/thread/thread.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/platform.hpp \ - /home/pblunsom/packages/include/boost/config/requires_threads.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_data.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/config.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/platform.hpp \ - /home/pblunsom/packages/include/boost/thread/exceptions.hpp \ - /home/pblunsom/packages/include/boost/config/abi_prefix.hpp \ - /home/pblunsom/packages/include/boost/config/abi_suffix.hpp \ - /home/pblunsom/packages/include/boost/enable_shared_from_this.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/enable_shared_from_this.hpp \ - /home/pblunsom/packages/include/boost/smart_ptr/weak_ptr.hpp \ - /home/pblunsom/packages/include/boost/thread/mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/locks.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/move.hpp \ - /home/pblunsom/packages/include/boost/thread/thread_time.hpp \ - /home/pblunsom/packages/include/boost/date_time/microsec_time_clock.hpp \ - /home/pblunsom/packages/include/boost/date_time/compiler_config.hpp \ - /home/pblunsom/packages/include/boost/date_time/locale_config.hpp \ - /home/pblunsom/packages/include/boost/date_time/c_time.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_clock.hpp \ - /home/pblunsom/packages/include/boost/date_time/filetime_functions.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/ptime.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_system.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_config.hpp \ - /home/pblunsom/packages/include/boost/config/no_tr1/cmath.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_duration.hpp \ - /home/pblunsom/packages/include/boost/operators.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/special_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_resolution_traits.hpp \ - /home/pblunsom/packages/include/boost/date_time/int_adapter.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/gregorian_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/date.hpp \ - /home/pblunsom/packages/include/boost/date_time/year_month_day.hpp \ - /home/pblunsom/packages/include/boost/date_time/period.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_calendar.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_weekday.hpp \ - /home/pblunsom/packages/include/boost/date_time/constrained_value.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_of.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_base_and_derived.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_defs.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_day_of_year.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian_calendar.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian_calendar.ipp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_ymd.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_day.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_year.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_month.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_duration_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_duration_types.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/greg_date.hpp \ - /home/pblunsom/packages/include/boost/date_time/adjust_functors.hpp \ - /home/pblunsom/packages/include/boost/date_time/wrapping_int.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_generators.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_clock_device.hpp \ - /home/pblunsom/packages/include/boost/date_time/date_iterator.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_system_split.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_system_counted.hpp \ - /home/pblunsom/packages/include/boost/date_time/time.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/date_duration_operators.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/posix_time_duration.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/time_period.hpp \ - /home/pblunsom/packages/include/boost/date_time/time_iterator.hpp \ - /home/pblunsom/packages/include/boost/date_time/dst_rules.hpp \ - /home/pblunsom/packages/include/boost/thread/xtime.hpp \ - /home/pblunsom/packages/include/boost/date_time/posix_time/conversion.hpp \ - /home/pblunsom/packages/include/boost/date_time/gregorian/conversion.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/timespec.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/pthread_mutex_scoped_lock.hpp \ - /home/pblunsom/packages/include/boost/optional.hpp \ - /home/pblunsom/packages/include/boost/optional/optional.hpp \ - /home/pblunsom/packages/include/boost/type_traits/type_with_alignment.hpp \ - /home/pblunsom/packages/include/boost/detail/reference_content.hpp \ - /home/pblunsom/packages/include/boost/type_traits/has_nothrow_copy.hpp \ - /home/pblunsom/packages/include/boost/none.hpp \ - /home/pblunsom/packages/include/boost/none_t.hpp \ - /home/pblunsom/packages/include/boost/utility/compare_pointees.hpp \ - /home/pblunsom/packages/include/boost/optional/optional_fwd.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/condition_variable_fwd.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_heap_alloc.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_heap_alloc.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_interruption.hpp \ - /home/pblunsom/packages/include/boost/thread/detail/thread_group.hpp \ - /home/pblunsom/packages/include/boost/thread/shared_mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/shared_mutex.hpp \ - /home/pblunsom/packages/include/boost/thread/condition_variable.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/condition_variable.hpp \ - /home/pblunsom/packages/include/boost/thread/pthread/thread_data.hpp \ - /home/pblunsom/packages/include/boost/thread/future.hpp \ - /home/pblunsom/packages/include/boost/exception_ptr.hpp \ - /home/pblunsom/packages/include/boost/exception/detail/exception_ptr.hpp \ - /home/pblunsom/packages/include/boost/type_traits/is_fundamental.hpp \ - /home/pblunsom/packages/include/boost/thread/condition.hpp timing.h \ - clock_gettime_stub.c contexts_corpus.hh contexts_lexer.h \ - ../../../decoder/dict.h \ - /home/pblunsom/packages/include/boost/functional/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/hash_fwd.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/float_functions.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/limits.hpp \ - /home/pblunsom/packages/include/boost/integer/static_log2.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/detail/hash_float_generic.hpp \ - /home/pblunsom/packages/include/boost/functional/hash/extensions.hpp \ - /home/pblunsom/packages/include/boost/detail/container_fwd.hpp \ - ../../../decoder/wordid.h gzstream.hh -clock_gettime_stub.o: clock_gettime_stub.c -gammadist.o: gammadist.c gammadist.h mt19937ar.h -mt19937ar.o: mt19937ar.c mt19937ar.h diff --git a/gi/pyp-topics/src/mpi-corpus.hh b/gi/pyp-topics/src/mpi-corpus.hh deleted file mode 100644 index f5c478a9..00000000 --- a/gi/pyp-topics/src/mpi-corpus.hh +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef _MPI_CORPUS_HH -#define _MPI_CORPUS_HH - -#include <vector> -#include <string> -#include <map> -#include <tr1/unordered_map> - -#include <boost/ptr_container/ptr_vector.hpp> -#include <boost/mpi/environment.hpp> -#include <boost/mpi/communicator.hpp> - -#include "contexts_corpus.hh" - - -//////////////////////////////////////////////////////////////// -// MPICorpus -//////////////////////////////////////////////////////////////// - -class MPICorpus : public ContextsCorpus { -public: - MPICorpus() : ContextsCorpus() { - boost::mpi::communicator world; - m_rank = world.rank(); - m_size = world.size(); - m_start = -1; - m_end = -1; - } - virtual ~MPICorpus() {} - - virtual unsigned read_contexts(const std::string &filename, - BackoffGenerator* backoff_gen=0, - bool filter_singeltons=false, - bool binary_contexts=false) { - unsigned result = ContextsCorpus::read_contexts(filename, backoff_gen, filter_singeltons, binary_contexts); - - if (m_rank == 0) std::cerr << "\tLoad balancing terms per mpi segment:" << std::endl; - float segment_size = num_terms() / m_size; - float term_threshold = segment_size; - int seen_terms = 0; - std::vector<int> end_points; - for (int i=0; i < num_documents(); ++i) { - seen_terms += m_documents.at(i).size(); - if (seen_terms >= term_threshold) { - end_points.push_back(i+1); - term_threshold += segment_size; - if (m_rank == 0) std::cerr << "\t\t" << i+1 << ": " << seen_terms << " terms, " << 100*seen_terms / (float)num_terms() << "%" << std::endl; - } - } - m_start = (m_rank == 0 ? 0 : end_points.at(m_rank-1)); - m_end = (m_rank == m_size-1 ? num_documents() : end_points.at(m_rank)); - - return result; - } - - void - bounds(int* start, int* end) const { - *start = m_start; - *end = m_end; - } - - - -protected: - int m_rank, m_size; - int m_start, m_end; -}; - -#endif // _MPI_CORPUS_HH diff --git a/gi/pyp-topics/src/mpi-pyp-topics.cc b/gi/pyp-topics/src/mpi-pyp-topics.cc deleted file mode 100644 index d6e22af6..00000000 --- a/gi/pyp-topics/src/mpi-pyp-topics.cc +++ /dev/null @@ -1,466 +0,0 @@ -#include <boost/mpi/communicator.hpp> - -#include "timing.h" -#include "mpi-pyp-topics.hh" - -//#include <boost/date_time/posix_time/posix_time_types.hpp> -void MPIPYPTopics::sample_corpus(const MPICorpus& corpus, int samples, - int freq_cutoff_start, int freq_cutoff_end, - int freq_cutoff_interval, - int max_contexts_per_document) { - Timer timer; - - //int documents = corpus.num_documents(); - /* - m_mpi_start = 0; - m_mpi_end = documents; - if (m_size != 1) { - assert(documents < std::numeric_limits<int>::max()); - m_mpi_start = (documents / m_size) * m_rank; - if (m_rank == m_size-1) m_mpi_end = documents; - else m_mpi_end = (documents / m_size)*(m_rank+1); - } - */ - corpus.bounds(&m_mpi_start, &m_mpi_end); - int local_documents = m_mpi_end - m_mpi_start; - - if (!m_backoff.get()) { - m_word_pyps.clear(); - m_word_pyps.push_back(MPIPYPs()); - } - - if (m_am_root) std::cerr << "\n Training with " << m_word_pyps.size()-1 << " backoff level" - << (m_word_pyps.size()>1 ? ":" : "s:") << std::endl; - - for (int i=0; i<(int)m_word_pyps.size(); ++i) { - m_word_pyps.at(i).reserve(m_num_topics); - for (int j=0; j<m_num_topics; ++j) - m_word_pyps.at(i).push_back(new MPIPYP<int>(0.5, 1.0)); - } - if (m_am_root) std::cerr << std::endl; - - m_document_pyps.reserve(local_documents); - //m_document_pyps.reserve(corpus.num_documents()); - //for (int j=0; j<corpus.num_documents(); ++j) - for (int j=0; j<local_documents; ++j) - m_document_pyps.push_back(new PYP<int>(0.5, 1.0)); - - m_topic_p0 = 1.0/m_num_topics; - m_term_p0 = 1.0/corpus.num_types(); - m_backoff_p0 = 1.0/corpus.num_documents(); - - if (m_am_root) std::cerr << " Documents: " << corpus.num_documents() << "(" - << local_documents << ")" << " Terms: " << corpus.num_types() << std::endl; - - int frequency_cutoff = freq_cutoff_start; - if (m_am_root) std::cerr << " Context frequency cutoff set to " << frequency_cutoff << std::endl; - - timer.Reset(); - // Initialisation pass - int document_id=0, topic_counter=0; - for (int i=0; i<local_documents; ++i) { - document_id = i+m_mpi_start; - - //for (Corpus::const_iterator corpusIt=corpus.begin(); - // corpusIt != corpus.end(); ++corpusIt, ++document_id) { - m_corpus_topics.push_back(DocumentTopics(corpus.at(document_id).size(), 0)); - - int term_index=0; - for (Document::const_iterator docIt=corpus.at(document_id).begin(); - docIt != corpus.at(document_id).end(); ++docIt, ++term_index) { - topic_counter++; - Term term = *docIt; - - // sample a new_topic - //int new_topic = (topic_counter % m_num_topics); - int freq = corpus.context_count(term); - int new_topic = -1; - if (freq > frequency_cutoff - && (!max_contexts_per_document || term_index < max_contexts_per_document)) { - new_topic = sample(i, term); - //new_topic = document_id % m_num_topics; - - // add the new topic to the PYPs - increment(term, new_topic); - - if (m_use_topic_pyp) { - F p0 = m_topic_pyp.prob(new_topic, m_topic_p0); - int table_delta = m_document_pyps.at(i).increment(new_topic, p0); - if (table_delta) - m_topic_pyp.increment(new_topic, m_topic_p0, rnd); - } - else m_document_pyps.at(i).increment(new_topic, m_topic_p0); - } - - m_corpus_topics.at(i).at(term_index) = new_topic; - } - } - - // Synchronise the topic->word counds across the processes. - synchronise(); - - if (m_am_root) std::cerr << " Initialized in " << timer.Elapsed() << " seconds\n"; - - int* randomDocIndices = new int[local_documents]; - for (int i = 0; i < local_documents; ++i) - randomDocIndices[i] = i; - - // Sampling phase - for (int curr_sample=0; curr_sample < samples; ++curr_sample) { - if (freq_cutoff_interval > 0 && curr_sample != 1 - && curr_sample % freq_cutoff_interval == 1 - && frequency_cutoff > freq_cutoff_end) { - frequency_cutoff--; - if (m_am_root) std::cerr << "\n Context frequency cutoff set to " << frequency_cutoff << std::endl; - } - - if (m_am_root) std::cerr << "\n -- Sample " << curr_sample << " "; std::cerr.flush(); - - // Randomize the corpus indexing array - int tmp; - int processed_terms=0; - for (int i = (local_documents-1); i > 0; --i) { - //i+1 since j \in [0,i] but rnd() \in [0,1) - int j = (int)(rnd() * (i+1)); - assert(j >= 0 && j <= i); - tmp = randomDocIndices[i]; - randomDocIndices[i] = randomDocIndices[j]; - randomDocIndices[j] = tmp; - } - - // for each document in the corpus - for (int rand_doc=0; rand_doc<local_documents; ++rand_doc) { - int doc_index = randomDocIndices[rand_doc]; - int document_id = doc_index + m_mpi_start; - const Document& doc = corpus.at(document_id); - - // for each term in the document - int term_index=0; - Document::const_iterator docEnd = doc.end(); - for (Document::const_iterator docIt=doc.begin(); - docIt != docEnd; ++docIt, ++term_index) { - - if (max_contexts_per_document && term_index > max_contexts_per_document) - break; - - Term term = *docIt; - int freq = corpus.context_count(term); - if (freq < frequency_cutoff) - continue; - - processed_terms++; - - // remove the prevous topic from the PYPs - int current_topic = m_corpus_topics.at(doc_index).at(term_index); - // a negative label mean that term hasn't been sampled yet - if (current_topic >= 0) { - decrement(term, current_topic); - - int table_delta = m_document_pyps.at(doc_index).decrement(current_topic); - if (m_use_topic_pyp && table_delta < 0) - m_topic_pyp.decrement(current_topic, rnd); - } - - // sample a new_topic - int new_topic = sample(doc_index, term); - - // add the new topic to the PYPs - m_corpus_topics.at(doc_index).at(term_index) = new_topic; - increment(term, new_topic); - - if (m_use_topic_pyp) { - F p0 = m_topic_pyp.prob(new_topic, m_topic_p0); - int table_delta = m_document_pyps.at(doc_index).increment(new_topic, p0); - if (table_delta) - m_topic_pyp.increment(new_topic, m_topic_p0, rnd); - } - else m_document_pyps.at(doc_index).increment(new_topic, m_topic_p0); - } - if (document_id && document_id % 10000 == 0) { - if (m_am_root) std::cerr << "."; std::cerr.flush(); - } - } - std::cerr << "|"; std::cerr.flush(); - - // Synchronise the topic->word counds across the processes. - synchronise(); - - if (m_am_root) std::cerr << " ||| sampled " << processed_terms << " terms."; - - if (curr_sample != 0 && curr_sample % 10 == 0) { - if (m_am_root) std::cerr << " ||| time=" << (timer.Elapsed() / 10.0) << " sec/sample" << std::endl; - timer.Reset(); - if (m_am_root) std::cerr << " ... Resampling hyperparameters"; std::cerr.flush(); - - // resample the hyperparamters - F log_p=0.0; - for (std::vector<MPIPYPs>::iterator levelIt=m_word_pyps.begin(); - levelIt != m_word_pyps.end(); ++levelIt) { - for (MPIPYPs::iterator pypIt=levelIt->begin(); - pypIt != levelIt->end(); ++pypIt) { - pypIt->resample_prior(rnd); - log_p += pypIt->log_restaurant_prob(); - } - } - - for (PYPs::iterator pypIt=m_document_pyps.begin(); - pypIt != m_document_pyps.end(); ++pypIt) { - pypIt->resample_prior(rnd); - log_p += pypIt->log_restaurant_prob(); - } - - if (m_use_topic_pyp) { - m_topic_pyp.resample_prior(rnd); - log_p += m_topic_pyp.log_restaurant_prob(); - } - - std::cerr.precision(10); - if (m_am_root) std::cerr << " ||| LLH=" << log_p << " ||| resampling time=" << timer.Elapsed() << " sec" << std::endl; - timer.Reset(); - - int k=0; - if (m_am_root) std::cerr << "Topics distribution: "; - std::cerr.precision(2); - for (MPIPYPs::iterator pypIt=m_word_pyps.front().begin(); - pypIt != m_word_pyps.front().end(); ++pypIt, ++k) { - if (m_am_root && k % 5 == 0) std::cerr << std::endl << '\t'; - if (m_am_root) std::cerr << "<" << k << ":" << pypIt->num_customers() << "," - << pypIt->num_types() << "," << m_topic_pyp.prob(k, m_topic_p0) << "> "; - } - std::cerr.precision(4); - if (m_am_root) std::cerr << std::endl; - } - } - delete [] randomDocIndices; -} - -void MPIPYPTopics::synchronise() { - // Synchronise the topic->word counds across the processes. - //for (std::vector<MPIPYPs>::iterator levelIt=m_word_pyps.begin(); - // levelIt != m_word_pyps.end(); ++levelIt) { -// std::vector<MPIPYPs>::iterator levelIt=m_word_pyps.begin(); -// { -// for (MPIPYPs::iterator pypIt=levelIt->begin(); pypIt != levelIt->end(); ++pypIt) { - for (size_t label=0; label < m_word_pyps.at(0).size(); ++label) { - MPIPYP<int>& pyp = m_word_pyps.at(0).at(label); - - //if (!m_am_root) boost::mpi::communicator().barrier(); - //std::cerr << "Before Sync Process " << m_rank << ":"; - //pyp.debug_info(std::cerr); std::cerr << std::endl; - //if (m_am_root) boost::mpi::communicator().barrier(); - - MPIPYP<int>::dish_delta_type delta; - pyp.synchronise(&delta); - - for (MPIPYP<int>::dish_delta_type::const_iterator it=delta.begin(); it != delta.end(); ++it) { - int count = it->second; - if (count > 0) - for (int i=0; i < count; ++i) increment(it->first, label); - if (count < 0) - for (int i=0; i > count; --i) decrement(it->first, label); - } - pyp.reset_deltas(); - - //if (!m_am_root) boost::mpi::communicator().barrier(); - //std::cerr << "After Sync Process " << m_rank << ":"; - //pyp.debug_info(std::cerr); std::cerr << std::endl; - //if (m_am_root) boost::mpi::communicator().barrier(); - } -// } - // Synchronise the hierarchical topic pyp - MPIPYP<int>::dish_delta_type topic_delta; - m_topic_pyp.synchronise(&topic_delta); - for (MPIPYP<int>::dish_delta_type::const_iterator it=topic_delta.begin(); it != topic_delta.end(); ++it) { - int count = it->second; - if (count > 0) - for (int i=0; i < count; ++i) - m_topic_pyp.increment(it->first, m_topic_p0, rnd); - if (count < 0) - for (int i=0; i > count; --i) - m_topic_pyp.decrement(it->first, rnd); - } - m_topic_pyp.reset_deltas(); -} - -void MPIPYPTopics::decrement(const Term& term, int topic, int level) { - //std::cerr << "MPIPYPTopics::decrement(" << term << "," << topic << "," << level << ")" << std::endl; - m_word_pyps.at(level).at(topic).decrement(term, rnd); - if (m_backoff.get()) { - Term backoff_term = (*m_backoff)[term]; - if (!m_backoff->is_null(backoff_term)) - decrement(backoff_term, topic, level+1); - } -} - -void MPIPYPTopics::increment(const Term& term, int topic, int level) { - //std::cerr << "MPIPYPTopics::increment(" << term << "," << topic << "," << level << ")" << std::endl; - m_word_pyps.at(level).at(topic).increment(term, word_pyps_p0(term, topic, level), rnd); - - if (m_backoff.get()) { - Term backoff_term = (*m_backoff)[term]; - if (!m_backoff->is_null(backoff_term)) - increment(backoff_term, topic, level+1); - } -} - -int MPIPYPTopics::sample(const DocumentId& doc, const Term& term) { - // First pass: collect probs - F sum=0.0; - std::vector<F> sums; - for (int k=0; k<m_num_topics; ++k) { - F p_w_k = prob(term, k); - - F topic_prob = m_topic_p0; - if (m_use_topic_pyp) topic_prob = m_topic_pyp.prob(k, m_topic_p0); - - //F p_k_d = m_document_pyps[doc].prob(k, topic_prob); - F p_k_d = m_document_pyps.at(doc).unnormalised_prob(k, topic_prob); - - sum += (p_w_k*p_k_d); - sums.push_back(sum); - } - // Second pass: sample a topic - F cutoff = rnd() * sum; - for (int k=0; k<m_num_topics; ++k) { - if (cutoff <= sums[k]) - return k; - } - std::cerr << cutoff << " " << sum << std::endl; - assert(false); -} - -MPIPYPTopics::F MPIPYPTopics::word_pyps_p0(const Term& term, int topic, int level) const { - //for (int i=0; i<level+1; ++i) std::cerr << " "; - //std::cerr << "MPIPYPTopics::word_pyps_p0(" << term << "," << topic << "," << level << ")" << std::endl; - - F p0 = m_term_p0; - if (m_backoff.get()) { - //static F fudge=m_backoff_p0; // TODO - - Term backoff_term = (*m_backoff)[term]; - if (!m_backoff->is_null(backoff_term)) { - assert (level < m_backoff->order()); - //p0 = (1.0/(double)m_backoff->terms_at_level(level))*prob(backoff_term, topic, level+1); - p0 = prob(backoff_term, topic, level+1); - } - else - p0 = m_term_p0; - } - //for (int i=0; i<level+1; ++i) std::cerr << " "; - //std::cerr << "MPIPYPTopics::word_pyps_p0(" << term << "," << topic << "," << level << ") = " << p0 << std::endl; - return p0; -} - -MPIPYPTopics::F MPIPYPTopics::prob(const Term& term, int topic, int level) const { - //for (int i=0; i<level+1; ++i) std::cerr << " "; - //std::cerr << "MPIPYPTopics::prob(" << term << "," << topic << "," << level << " " << factor << ")" << std::endl; - - F p0 = word_pyps_p0(term, topic, level); - F p_w_k = m_word_pyps.at(level).at(topic).prob(term, p0); - - //for (int i=0; i<level+1; ++i) std::cerr << " "; - //std::cerr << "MPIPYPTopics::prob(" << term << "," << topic << "," << level << ") = " << p_w_k << std::endl; - - return p_w_k; -} - -int MPIPYPTopics::max_topic() const { - if (!m_use_topic_pyp) - return -1; - - F current_max=0.0; - int current_topic=-1; - for (int k=0; k<m_num_topics; ++k) { - F prob = m_topic_pyp.prob(k, m_topic_p0); - if (prob > current_max) { - current_max = prob; - current_topic = k; - } - } - assert(current_topic >= 0); - assert(current_max >= 0); - return current_max; -} - -std::pair<int,MPIPYPTopics::F> MPIPYPTopics::max(const DocumentId& true_doc) const { - //std::cerr << "MPIPYPTopics::max(" << doc << "," << term << ")" << std::endl; - // collect probs - F current_max=0.0; - DocumentId local_doc = true_doc - m_mpi_start; - int current_topic=-1; - for (int k=0; k<m_num_topics; ++k) { - //F p_w_k = prob(term, k); - - F topic_prob = m_topic_p0; - if (m_use_topic_pyp) - topic_prob = m_topic_pyp.prob(k, m_topic_p0); - - F prob = 0; - if (local_doc < 0) prob = topic_prob; - else prob = m_document_pyps.at(local_doc).prob(k, topic_prob); - - if (prob > current_max) { - current_max = prob; - current_topic = k; - } - } - assert(current_topic >= 0); - assert(current_max >= 0); - return std::make_pair(current_topic, current_max); -} - -std::pair<int,MPIPYPTopics::F> MPIPYPTopics::max(const DocumentId& true_doc, const Term& term) const { - //std::cerr << "MPIPYPTopics::max(" << doc << "," << term << ")" << std::endl; - // collect probs - F current_max=0.0; - DocumentId local_doc = true_doc - m_mpi_start; - int current_topic=-1; - for (int k=0; k<m_num_topics; ++k) { - F p_w_k = prob(term, k); - - F topic_prob = m_topic_p0; - if (m_use_topic_pyp) - topic_prob = m_topic_pyp.prob(k, m_topic_p0); - - F p_k_d = 0; - if (local_doc < 0) p_k_d = topic_prob; - else p_k_d = m_document_pyps.at(local_doc).prob(k, topic_prob); - - F prob = (p_w_k*p_k_d); - if (prob > current_max) { - current_max = prob; - current_topic = k; - } - } - assert(current_topic >= 0); - assert(current_max >= 0); - return std::make_pair(current_topic, current_max); -} - -std::ostream& MPIPYPTopics::print_document_topics(std::ostream& out) const { - for (CorpusTopics::const_iterator corpusIt=m_corpus_topics.begin(); - corpusIt != m_corpus_topics.end(); ++corpusIt) { - int term_index=0; - for (DocumentTopics::const_iterator docIt=corpusIt->begin(); - docIt != corpusIt->end(); ++docIt, ++term_index) { - if (term_index) out << " "; - out << *docIt; - } - out << std::endl; - } - return out; -} - -std::ostream& MPIPYPTopics::print_topic_terms(std::ostream& out) const { - for (PYPs::const_iterator pypsIt=m_word_pyps.front().begin(); - pypsIt != m_word_pyps.front().end(); ++pypsIt) { - int term_index=0; - for (PYP<int>::const_iterator termIt=pypsIt->begin(); - termIt != pypsIt->end(); ++termIt, ++term_index) { - if (term_index) out << " "; - out << termIt->first << ":" << termIt->second; - } - out << std::endl; - } - return out; -} diff --git a/gi/pyp-topics/src/mpi-pyp-topics.hh b/gi/pyp-topics/src/mpi-pyp-topics.hh deleted file mode 100644 index d96bc4e5..00000000 --- a/gi/pyp-topics/src/mpi-pyp-topics.hh +++ /dev/null @@ -1,106 +0,0 @@ -#ifndef MPI_PYP_TOPICS_HH -#define MPI_PYP_TOPICS_HH - -#include <vector> -#include <iostream> - -#include <boost/ptr_container/ptr_vector.hpp> -#include <boost/random/uniform_real.hpp> -#include <boost/random/variate_generator.hpp> -#include <boost/random/mersenne_twister.hpp> -#include <boost/random/inversive_congruential.hpp> -#include <boost/random/linear_congruential.hpp> -#include <boost/random/lagged_fibonacci.hpp> -#include <boost/mpi/environment.hpp> -#include <boost/mpi/communicator.hpp> - - -#include "mpi-pyp.hh" -#include "mpi-corpus.hh" - -class MPIPYPTopics { -public: - typedef std::vector<int> DocumentTopics; - typedef std::vector<DocumentTopics> CorpusTopics; - typedef double F; - -public: - MPIPYPTopics(int num_topics, bool use_topic_pyp=false, unsigned long seed = 0) - : m_num_topics(num_topics), m_word_pyps(1), - m_topic_pyp(0.5,1.0), m_use_topic_pyp(use_topic_pyp), - m_seed(seed), - uni_dist(0,1), rng(seed == 0 ? (unsigned long)this : seed), - rnd(rng, uni_dist), m_mpi_start(-1), m_mpi_end(-1) { - boost::mpi::communicator m_world; - m_rank = m_world.rank(); - m_size = m_world.size(); - m_am_root = (m_rank == 0); - } - - void sample_corpus(const MPICorpus& corpus, int samples, - int freq_cutoff_start=0, int freq_cutoff_end=0, - int freq_cutoff_interval=0, - int max_contexts_per_document=0); - - int sample(const DocumentId& doc, const Term& term); - std::pair<int,F> max(const DocumentId& doc, const Term& term) const; - std::pair<int,F> max(const DocumentId& doc) const; - int max_topic() const; - - void set_backoff(const std::string& filename) { - m_backoff.reset(new TermBackoff); - m_backoff->read(filename); - m_word_pyps.clear(); - m_word_pyps.resize(m_backoff->order(), MPIPYPs()); - } - void set_backoff(TermBackoffPtr backoff) { - m_backoff = backoff; - m_word_pyps.clear(); - m_word_pyps.resize(m_backoff->order(), MPIPYPs()); - } - - F prob(const Term& term, int topic, int level=0) const; - void decrement(const Term& term, int topic, int level=0); - void increment(const Term& term, int topic, int level=0); - - std::ostream& print_document_topics(std::ostream& out) const; - std::ostream& print_topic_terms(std::ostream& out) const; - - void synchronise(); - -private: - F word_pyps_p0(const Term& term, int topic, int level) const; - - int m_num_topics; - F m_term_p0, m_topic_p0, m_backoff_p0; - - CorpusTopics m_corpus_topics; - typedef boost::ptr_vector< PYP<int> > PYPs; - typedef boost::ptr_vector< MPIPYP<int> > MPIPYPs; - PYPs m_document_pyps; - std::vector<MPIPYPs> m_word_pyps; - MPIPYP<int> m_topic_pyp; - bool m_use_topic_pyp; - - unsigned long m_seed; - - //typedef boost::mt19937 base_generator_type; - //typedef boost::hellekalek1995 base_generator_type; - typedef boost::lagged_fibonacci607 base_generator_type; - typedef boost::uniform_real<> uni_dist_type; - typedef boost::variate_generator<base_generator_type&, uni_dist_type> gen_type; - - uni_dist_type uni_dist; - base_generator_type rng; //this gets the seed - gen_type rnd; //instantiate: rnd(rng, uni_dist) - //call: rnd() generates uniform on [0,1) - - TermBackoffPtr m_backoff; - - boost::mpi::communicator m_world; - bool m_am_root; - int m_rank, m_size; - int m_mpi_start, m_mpi_end; -}; - -#endif // PYP_TOPICS_HH diff --git a/gi/pyp-topics/src/mpi-pyp.hh b/gi/pyp-topics/src/mpi-pyp.hh deleted file mode 100644 index c2341b9e..00000000 --- a/gi/pyp-topics/src/mpi-pyp.hh +++ /dev/null @@ -1,447 +0,0 @@ -#ifndef _mpipyp_hh -#define _mpipyp_hh - -#include <math.h> -#include <map> -#include <tr1/unordered_map> -//#include <google/sparse_hash_map> - -#include <boost/random/uniform_real.hpp> -#include <boost/random/variate_generator.hpp> -#include <boost/random/mersenne_twister.hpp> -#include <boost/tuple/tuple.hpp> -#include <boost/serialization/map.hpp> -#include <boost/mpi.hpp> -#include <boost/mpi/environment.hpp> -#include <boost/mpi/communicator.hpp> -#include <boost/mpi/operations.hpp> - - -#include "pyp.hh" - -// -// Pitman-Yor process with customer and table tracking -// - -template <typename Dish, typename Hash=std::tr1::hash<Dish> > -class MPIPYP : public PYP<Dish, Hash> { -public: - typedef std::map<Dish, int> dish_delta_type; - - MPIPYP(double a, double b, Hash hash=Hash()); - - template < typename Uniform01 > - int increment(Dish d, double p0, Uniform01& rnd); - template < typename Uniform01 > - int decrement(Dish d, Uniform01& rnd); - - void clear(); - void reset_deltas(); - - void synchronise(dish_delta_type* result); - -private: - typedef std::map<Dish, typename PYP<Dish,Hash>::TableCounter> table_delta_type; - - dish_delta_type m_count_delta; - table_delta_type m_table_delta; -}; - -template <typename Dish, typename Hash> -MPIPYP<Dish,Hash>::MPIPYP(double a, double b, Hash h) -: PYP<Dish,Hash>(a, b, 0, h) {} - -template <typename Dish, typename Hash> - template <typename Uniform01> -int -MPIPYP<Dish,Hash>::increment(Dish dish, double p0, Uniform01& rnd) { - //std::cerr << "-----INCREMENT DISH " << dish << std::endl; - int delta = 0; - int table_joined=-1; - typename PYP<Dish,Hash>::TableCounter &tc = PYP<Dish,Hash>::_dish_tables[dish]; - - // seated on a new or existing table? - int c = PYP<Dish,Hash>::count(dish); - int t = PYP<Dish,Hash>::num_tables(dish); - int T = PYP<Dish,Hash>::num_tables(); - double& a = PYP<Dish,Hash>::_a; - double& b = PYP<Dish,Hash>::_b; - double pshare = (c > 0) ? (c - a*t) : 0.0; - double pnew = (b + a*T) * p0; - if (pshare < 0.0) { - std::cerr << pshare << " " << c << " " << a << " " << t << std::endl; - assert(false); - } - - if (rnd() < pnew / (pshare + pnew)) { - // assign to a new table - tc.tables += 1; - tc.table_histogram[1] += 1; - PYP<Dish,Hash>::_total_tables += 1; - delta = 1; - table_joined = 1; - } - else { - // randomly assign to an existing table - // remove constant denominator from inner loop - double r = rnd() * (c - a*t); - for (std::map<int,int>::iterator - hit = tc.table_histogram.begin(); - hit != tc.table_histogram.end(); ++hit) { - r -= ((hit->first - a) * hit->second); - if (r <= 0) { - tc.table_histogram[hit->first+1] += 1; - hit->second -= 1; - table_joined = hit->first+1; - if (hit->second == 0) - tc.table_histogram.erase(hit); - break; - } - } - if (r > 0) { - std::cerr << r << " " << c << " " << a << " " << t << std::endl; - assert(false); - } - delta = 0; - } - - std::tr1::unordered_map<Dish,int,Hash>::operator[](dish) += 1; - //google::sparse_hash_map<Dish,int,Hash>::operator[](dish) += 1; - PYP<Dish,Hash>::_total_customers += 1; - - // MPI Delta handling - // track the customer entering - typename dish_delta_type::iterator customer_it; - bool customer_insert_result; - boost::tie(customer_it, customer_insert_result) - = m_count_delta.insert(std::make_pair(dish,0)); - - customer_it->second += 1; - if (customer_it->second == 0) - m_count_delta.erase(customer_it); - - // increment the histogram bar for the table joined - /* - typename PYP<Dish,Hash>::TableCounter &delta_tc = m_table_delta[dish]; - - std::map<int,int> &histogram = delta_tc.table_histogram; - assert (table_joined > 0); - - typename std::map<int,int>::iterator table_it; bool table_insert_result; - boost::tie(table_it, table_insert_result) = histogram.insert(std::make_pair(table_joined,0)); - table_it->second += 1; - if (delta == 0) { - // decrement the histogram bar for the table left - typename std::map<int,int>::iterator left_table_it; - boost::tie(left_table_it, table_insert_result) - = histogram.insert(std::make_pair(table_joined-1,0)); - left_table_it->second -= 1; - if (left_table_it->second == 0) histogram.erase(left_table_it); - } - else delta_tc.tables += 1; - - if (table_it->second == 0) histogram.erase(table_it); - - //std::cerr << "Added (" << delta << ") " << dish << " to table " << table_joined << "\n"; - //std::cerr << "Dish " << dish << " has " << count(dish) << " customers, and is sitting at " << PYP<Dish,Hash>::num_tables(dish) << " tables.\n"; - //for (std::map<int,int>::const_iterator - // hit = delta_tc.table_histogram.begin(); - // hit != delta_tc.table_histogram.end(); ++hit) { - // std::cerr << " " << hit->second << " tables with " << hit->first << " customers." << std::endl; - //} - //std::cerr << "Added (" << delta << ") " << dish << " to table " << table_joined << "\n"; - //std::cerr << "Dish " << dish << " has " << count(dish) << " customers, and is sitting at " << PYP<Dish,Hash>::num_tables(dish) << " tables.\n"; - int x_num_customers=0, x_num_table=0; - for (std::map<int,int>::const_iterator - hit = delta_tc.table_histogram.begin(); - hit != delta_tc.table_histogram.end(); ++hit) { - x_num_table += hit->second; - x_num_customers += (hit->second*hit->first); - } - int tmp_c = PYP<Dish,Hash>::count(dish); - int tmp_t = PYP<Dish,Hash>::num_tables(dish); - assert (x_num_customers <= tmp_c); - assert (x_num_table <= tmp_t); - - if (delta_tc.table_histogram.empty()) { - assert (delta_tc.tables == 0); - m_table_delta.erase(dish); - } - */ - - //PYP<Dish,Hash>::debug_info(std::cerr); - //std::cerr << " Dish " << dish << " has count " << PYP<Dish,Hash>::count(dish) << " tables " << PYP<Dish,Hash>::num_tables(dish) << std::endl; - - return delta; -} - -template <typename Dish, typename Hash> - template <typename Uniform01> -int -MPIPYP<Dish,Hash>::decrement(Dish dish, Uniform01& rnd) -{ - //std::cerr << "-----DECREMENT DISH " << dish << std::endl; - typename std::tr1::unordered_map<Dish, int>::iterator dcit = find(dish); - //typename google::sparse_hash_map<Dish, int>::iterator dcit = find(dish); - if (dcit == PYP<Dish,Hash>::end()) { - std::cerr << dish << std::endl; - assert(false); - } - - int delta = 0, table_left=-1; - - typename std::tr1::unordered_map<Dish, typename PYP<Dish,Hash>::TableCounter>::iterator dtit - = PYP<Dish,Hash>::_dish_tables.find(dish); - //typename google::sparse_hash_map<Dish, TableCounter>::iterator dtit = _dish_tables.find(dish); - if (dtit == PYP<Dish,Hash>::_dish_tables.end()) { - std::cerr << dish << std::endl; - assert(false); - } - typename PYP<Dish,Hash>::TableCounter &tc = dtit->second; - - double r = rnd() * PYP<Dish,Hash>::count(dish); - for (std::map<int,int>::iterator hit = tc.table_histogram.begin(); - hit != tc.table_histogram.end(); ++hit) { - r -= (hit->first * hit->second); - if (r <= 0) { - table_left = hit->first; - if (hit->first > 1) { - tc.table_histogram[hit->first-1] += 1; - } - else { - delta = -1; - tc.tables -= 1; - PYP<Dish,Hash>::_total_tables -= 1; - } - - hit->second -= 1; - if (hit->second == 0) tc.table_histogram.erase(hit); - break; - } - } - if (r > 0) { - std::cerr << r << " " << PYP<Dish,Hash>::count(dish) << " " << PYP<Dish,Hash>::_a << " " - << PYP<Dish,Hash>::num_tables(dish) << std::endl; - assert(false); - } - - // remove the customer - dcit->second -= 1; - PYP<Dish,Hash>::_total_customers -= 1; - assert(dcit->second >= 0); - if (dcit->second == 0) { - PYP<Dish,Hash>::erase(dcit); - PYP<Dish,Hash>::_dish_tables.erase(dtit); - } - - // MPI Delta processing - typename dish_delta_type::iterator it; - bool insert_result; - boost::tie(it, insert_result) = m_count_delta.insert(std::make_pair(dish,0)); - it->second -= 1; - if (it->second == 0) m_count_delta.erase(it); - - assert (table_left > 0); - typename PYP<Dish,Hash>::TableCounter& delta_tc = m_table_delta[dish]; - if (table_left > 1) { - std::map<int,int>::iterator tit; - boost::tie(tit, insert_result) = delta_tc.table_histogram.insert(std::make_pair(table_left-1,0)); - tit->second += 1; - if (tit->second == 0) delta_tc.table_histogram.erase(tit); - } - else delta_tc.tables -= 1; - - std::map<int,int>::iterator tit; - boost::tie(tit, insert_result) = delta_tc.table_histogram.insert(std::make_pair(table_left,0)); - tit->second -= 1; - if (tit->second == 0) delta_tc.table_histogram.erase(tit); - - // std::cerr << "Dish " << dish << " has " << count(dish) << " customers, and is sitting at " << PYP<Dish,Hash>::num_tables(dish) << " tables.\n"; - // for (std::map<int,int>::const_iterator - // hit = delta_tc.table_histogram.begin(); - // hit != delta_tc.table_histogram.end(); ++hit) { - // std::cerr << " " << hit->second << " tables with " << hit->first << " customers." << std::endl; - // } - int x_num_customers=0, x_num_table=0; - for (std::map<int,int>::const_iterator - hit = delta_tc.table_histogram.begin(); - hit != delta_tc.table_histogram.end(); ++hit) { - x_num_table += hit->second; - x_num_customers += (hit->second*hit->first); - } - int tmp_c = PYP<Dish,Hash>::count(dish); - int tmp_t = PYP<Dish,Hash>::num_tables(dish); - assert (x_num_customers <= tmp_c); - assert (x_num_table <= tmp_t); - - if (delta_tc.table_histogram.empty()) { - // std::cerr << " DELETING " << dish << std::endl; - assert (delta_tc.tables == 0); - m_table_delta.erase(dish); - } - - //PYP<Dish,Hash>::debug_info(std::cerr); - //std::cerr << " Dish " << dish << " has count " << PYP<Dish,Hash>::count(dish) << " tables " << PYP<Dish,Hash>::num_tables(dish) << std::endl; - return delta; -} - -template <typename Dish, typename Hash> -void -MPIPYP<Dish,Hash>::clear() { - PYP<Dish,Hash>::clear(); - reset_deltas(); -} - -template <typename Dish, typename Hash> -void -MPIPYP<Dish,Hash>::reset_deltas() { - m_count_delta.clear(); - m_table_delta.clear(); -} - -template <typename Dish> -struct sum_maps { - typedef std::map<Dish,int> map_type; - map_type& operator() (map_type& l, map_type const & r) const { - for (typename map_type::const_iterator it=r.begin(); it != r.end(); it++) - l[it->first] += it->second; - return l; - } -}; - -template <typename Dish> -struct subtract_maps { - typedef std::map<Dish,int> map_type; - map_type& operator() (map_type& l, map_type const & r) const { - for (typename map_type::const_iterator it=r.begin(); it != r.end(); it++) - l[it->first] -= it->second; - return l; - } -}; - -// Needed Boost definitions -namespace boost { - namespace mpi { - template <> - struct is_commutative< sum_maps<int>, std::map<int,int> > : mpl::true_ {}; - } - - namespace serialization { - template<class Archive> - void serialize(Archive & ar, PYP<int>::TableCounter& t, const unsigned int version) { - ar & t.table_histogram; - ar & t.tables; - } - - } // namespace serialization -} // namespace boost - -template <typename A, typename B, typename C> -struct triple { - triple() {} - triple(const A& a, const B& b, const C& c) : first(a), second(b), third(c) {} - A first; - B second; - C third; - - template<class Archive> - void serialize(Archive &ar, const unsigned int version){ - ar & first; - ar & second; - ar & third; - } -}; - -BOOST_IS_BITWISE_SERIALIZABLE(MPIPYP<int>::dish_delta_type) -BOOST_CLASS_TRACKING(MPIPYP<int>::dish_delta_type,track_never) - -template <typename Dish, typename Hash> -void -MPIPYP<Dish,Hash>::synchronise(dish_delta_type* result) { - boost::mpi::communicator world; - //int rank = world.rank(), size = world.size(); - - boost::mpi::all_reduce(world, m_count_delta, *result, sum_maps<Dish>()); - subtract_maps<Dish>()(*result, m_count_delta); - -/* - // communicate the customer count deltas - dish_delta_type global_dish_delta; - boost::mpi::all_reduce(world, m_count_delta, global_dish_delta, sum_maps<Dish>()); - - // update this restaurant - for (typename dish_delta_type::const_iterator it=global_dish_delta.begin(); - it != global_dish_delta.end(); ++it) { - int global_delta = it->second - m_count_delta[it->first]; - if (global_delta == 0) continue; - typename std::tr1::unordered_map<Dish,int,Hash>::iterator dit; bool inserted; - boost::tie(dit, inserted) - = std::tr1::unordered_map<Dish,int,Hash>::insert(std::make_pair(it->first, 0)); - dit->second += global_delta; - assert(dit->second >= 0); - if (dit->second == 0) { - std::tr1::unordered_map<Dish,int,Hash>::erase(dit); - } - - PYP<Dish,Hash>::_total_customers += (it->second - m_count_delta[it->first]); - int tmp = PYP<Dish,Hash>::_total_customers; - assert(tmp >= 0); - //std::cerr << "Process " << rank << " adding " << (it->second - m_count_delta[it->first]) << " of customer " << it->first << std::endl; - } -*/ -/* - // communicate the table count deltas - for (int process = 0; process < size; ++process) { - typename std::vector< triple<Dish, int, int> > message; - if (rank == process) { - // broadcast deltas - for (typename table_delta_type::const_iterator dish_it=m_table_delta.begin(); - dish_it != m_table_delta.end(); ++dish_it) { - //assert (dish_it->second.tables > 0); - for (std::map<int,int>::const_iterator it=dish_it->second.table_histogram.begin(); - it != dish_it->second.table_histogram.end(); ++it) { - triple<Dish, int, int> m(dish_it->first, it->first, it->second); - message.push_back(m); - } - // append a special message with the total table delta for this dish - triple<Dish, int, int> m(dish_it->first, -1, dish_it->second.tables); - message.push_back(m); - } - boost::mpi::broadcast(world, message, process); - } - else { - // receive deltas - boost::mpi::broadcast(world, message, process); - for (typename std::vector< triple<Dish, int, int> >::const_iterator it=message.begin(); it != message.end(); ++it) { - typename PYP<Dish,Hash>::TableCounter& tc = PYP<Dish,Hash>::_dish_tables[it->first]; - if (it->second >= 0) { - std::map<int,int>::iterator tit; bool inserted; - boost::tie(tit, inserted) = tc.table_histogram.insert(std::make_pair(it->second, 0)); - tit->second += it->third; - if (tit->second < 0) { - std::cerr << tit->first << " " << tit->second << " " << it->first << " " << it->second << " " << it->third << std::endl; - assert(tit->second >= 0); - } - if (tit->second == 0) { - tc.table_histogram.erase(tit); - } - } - else { - tc.tables += it->third; - PYP<Dish,Hash>::_total_tables += it->third; - assert(tc.tables >= 0); - if (tc.tables == 0) assert(tc.table_histogram.empty()); - if (tc.table_histogram.empty()) { - assert (tc.tables == 0); - PYP<Dish,Hash>::_dish_tables.erase(it->first); - } - } - } - } - } -*/ - -// reset_deltas(); -} - -#endif diff --git a/gi/pyp-topics/src/mpi-train-contexts.cc b/gi/pyp-topics/src/mpi-train-contexts.cc deleted file mode 100644 index e05e0eac..00000000 --- a/gi/pyp-topics/src/mpi-train-contexts.cc +++ /dev/null @@ -1,201 +0,0 @@ -// STL -#include <iostream> -#include <fstream> -#include <algorithm> -#include <iterator> - -// Boost -#include <boost/program_options/parsers.hpp> -#include <boost/program_options/variables_map.hpp> -#include <boost/scoped_ptr.hpp> -#include <boost/mpi/environment.hpp> -#include <boost/mpi/communicator.hpp> -#include <boost/lexical_cast.hpp> - -// Local -#include "mpi-pyp-topics.hh" -#include "corpus.hh" -#include "mpi-corpus.hh" -#include "gzstream.hh" - -static const char *REVISION = "$Rev: 170 $"; - -// Namespaces -using namespace boost; -using namespace boost::program_options; -using namespace std; - -int main(int argc, char **argv) -{ - mpi::environment env(argc, argv); - mpi::communicator world; - int rank = world.rank(); - bool am_root = (rank==0); - if (am_root) cout << "Pitman Yor topic models: Copyright 2010 Phil Blunsom\n"; - if (am_root) std::cout << "I am process " << world.rank() << " of " << world.size() << "." << std::endl; - if (am_root) cout << REVISION << '\n' <<endl; - - //////////////////////////////////////////////////////////////////////////////////////////// - // Command line processing - variables_map vm; - - // Command line processing - { - options_description cmdline_specific("Command line specific options"); - cmdline_specific.add_options() - ("help,h", "print help message") - ("config,c", value<string>(), "config file specifying additional command line options") - ; - options_description config_options("Allowed options"); - config_options.add_options() - ("help,h", "print help message") - ("data,d", value<string>(), "file containing the documents and context terms") - ("topics,t", value<int>()->default_value(50), "number of topics") - ("document-topics-out,o", value<string>(), "file to write the document topics to") - ("default-topics-out", value<string>(), "file to write default term topic assignments.") - ("topic-words-out,w", value<string>(), "file to write the topic word distribution to") - ("samples,s", value<int>()->default_value(10), "number of sampling passes through the data") - ("backoff-type", value<string>(), "backoff type: none|simple") -// ("filter-singleton-contexts", "filter singleton contexts") - ("hierarchical-topics", "Use a backoff hierarchical PYP as the P0 for the document topics distribution.") - ("binary-counts,b", "Use binary rather than integer counts for contexts.") - ("freq-cutoff-start", value<int>()->default_value(0), "initial frequency cutoff.") - ("freq-cutoff-end", value<int>()->default_value(0), "final frequency cutoff.") - ("freq-cutoff-interval", value<int>()->default_value(0), "number of iterations between frequency decrement.") - ("max-contexts-per-document", value<int>()->default_value(0), "Only sample the n most frequent contexts for a document.") - ; - - cmdline_specific.add(config_options); - - store(parse_command_line(argc, argv, cmdline_specific), vm); - notify(vm); - - if (vm.count("config") > 0) { - ifstream config(vm["config"].as<string>().c_str()); - store(parse_config_file(config, config_options), vm); - } - - if (vm.count("help")) { - cout << cmdline_specific << "\n"; - return 1; - } - } - //////////////////////////////////////////////////////////////////////////////////////////// - - if (!vm.count("data")) { - cerr << "Please specify a file containing the data." << endl; - return 1; - } - - // seed the random number generator: 0 = automatic, specify value otherwise - unsigned long seed = 0; - MPIPYPTopics model(vm["topics"].as<int>(), vm.count("hierarchical-topics"), seed); - - // read the data - BackoffGenerator* backoff_gen=0; - if (vm.count("backoff-type")) { - if (vm["backoff-type"].as<std::string>() == "none") { - backoff_gen = 0; - } - else if (vm["backoff-type"].as<std::string>() == "simple") { - backoff_gen = new SimpleBackoffGenerator(); - } - else { - cerr << "Backoff type (--backoff-type) must be one of none|simple." <<endl; - return(1); - } - } - - //ContextsCorpus contexts_corpus; - MPICorpus contexts_corpus; - contexts_corpus.read_contexts(vm["data"].as<string>(), backoff_gen, /*vm.count("filter-singleton-contexts")*/ false, vm.count("binary-counts")); - int mpi_start = 0, mpi_end = 0; - contexts_corpus.bounds(&mpi_start, &mpi_end); - std::cerr << "\tProcess " << rank << " has documents " << mpi_start << " -> " << mpi_end << "." << std::endl; - - model.set_backoff(contexts_corpus.backoff_index()); - - if (backoff_gen) - delete backoff_gen; - - // train the sampler - model.sample_corpus(contexts_corpus, vm["samples"].as<int>(), - vm["freq-cutoff-start"].as<int>(), - vm["freq-cutoff-end"].as<int>(), - vm["freq-cutoff-interval"].as<int>(), - vm["max-contexts-per-document"].as<int>()); - - if (vm.count("document-topics-out")) { - std::ofstream documents_out((vm["document-topics-out"].as<string>() + ".pyp-process-" + boost::lexical_cast<std::string>(rank)).c_str()); - //int documents = contexts_corpus.num_documents(); - /* - int mpi_start = 0, mpi_end = documents; - if (world.size() != 1) { - mpi_start = (documents / world.size()) * rank; - if (rank == world.size()-1) mpi_end = documents; - else mpi_end = (documents / world.size())*(rank+1); - } - */ - - map<int,int> all_terms; - for (int document_id=mpi_start; document_id<mpi_end; ++document_id) { - assert (document_id < contexts_corpus.num_documents()); - const Document& doc = contexts_corpus.at(document_id); - vector<int> unique_terms; - for (Document::const_iterator docIt=doc.begin(); docIt != doc.end(); ++docIt) { - if (unique_terms.empty() || *docIt != unique_terms.back()) - unique_terms.push_back(*docIt); - // increment this terms frequency - pair<map<int,int>::iterator,bool> insert_result = all_terms.insert(make_pair(*docIt,1)); - if (!insert_result.second) - all_terms[*docIt] = all_terms[*docIt] + 1; - } - documents_out << contexts_corpus.key(document_id) << '\t'; - documents_out << model.max(document_id).first << " " << doc.size() << " ||| "; - for (std::vector<int>::const_iterator termIt=unique_terms.begin(); termIt != unique_terms.end(); ++termIt) { - if (termIt != unique_terms.begin()) - documents_out << " ||| "; - vector<std::string> strings = contexts_corpus.context2string(*termIt); - copy(strings.begin(), strings.end(),ostream_iterator<std::string>(documents_out, " ")); - std::pair<int,MPIPYPTopics::F> maxinfo = model.max(document_id, *termIt); - documents_out << "||| C=" << maxinfo.first << " P=" << maxinfo.second; - } - documents_out <<endl; - } - documents_out.close(); - world.barrier(); - - if (am_root) { - ogzstream root_documents_out(vm["document-topics-out"].as<string>().c_str()); - for (int p=0; p < world.size(); ++p) { - std::string rank_p_prefix((vm["document-topics-out"].as<string>() + ".pyp-process-" + boost::lexical_cast<std::string>(p)).c_str()); - std::ifstream rank_p_trees_istream(rank_p_prefix.c_str(), std::ios_base::binary); - root_documents_out << rank_p_trees_istream.rdbuf(); - rank_p_trees_istream.close(); - remove((rank_p_prefix).c_str()); - } - root_documents_out.close(); - } - - if (am_root && vm.count("default-topics-out")) { - ofstream default_topics(vm["default-topics-out"].as<string>().c_str()); - default_topics << model.max_topic() <<endl; - for (std::map<int,int>::const_iterator termIt=all_terms.begin(); termIt != all_terms.end(); ++termIt) { - vector<std::string> strings = contexts_corpus.context2string(termIt->first); - default_topics << model.max(-1, termIt->first).first << " ||| " << termIt->second << " ||| "; - copy(strings.begin(), strings.end(),ostream_iterator<std::string>(default_topics, " ")); - default_topics <<endl; - } - } - } - - if (am_root && vm.count("topic-words-out")) { - ogzstream topics_out(vm["topic-words-out"].as<string>().c_str()); - model.print_topic_terms(topics_out); - topics_out.close(); - } - - cout <<endl; - - return 0; -} diff --git a/gi/pyp-topics/src/mt19937ar.c b/gi/pyp-topics/src/mt19937ar.c deleted file mode 100644 index 6551ea39..00000000 --- a/gi/pyp-topics/src/mt19937ar.c +++ /dev/null @@ -1,194 +0,0 @@ -/* - A C-program for MT19937, with initialization improved 2002/1/26. - Coded by Takuji Nishimura and Makoto Matsumoto. - - Before using, initialize the state by using mt_init_genrand(seed) - or mt_init_by_array(init_key, key_length). - - Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. The names of its contributors may not be used to endorse or promote - products derived from this software without specific prior written - permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - Any feedback is very welcome. - http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html - email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) -*/ - -#include "mt19937ar.h" /* XXX MJ 17th March 2006 */ - -/* Period parameters */ -#define N 624 -#define M 397 -#define MATRIX_A 0x9908b0dfUL /* constant vector a */ -#define UPPER_MASK 0x80000000UL /* most significant w-r bits */ -#define LOWER_MASK 0x7fffffffUL /* least significant r bits */ - -static unsigned long mt[N]; /* the array for the state vector */ -static int mti=N+1; /* mti==N+1 means mt[N] is not initialized */ - -/* initializes mt[N] with a seed */ -void mt_init_genrand(unsigned long s) -{ - mt[0]= s & 0xffffffffUL; - for (mti=1; mti<N; mti++) { - mt[mti] = - (1812433253UL * (mt[mti-1] ^ (mt[mti-1] >> 30)) + mti); - /* See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier. */ - /* In the previous versions, MSBs of the seed affect */ - /* only MSBs of the array mt[]. */ - /* 2002/01/09 modified by Makoto Matsumoto */ - mt[mti] &= 0xffffffffUL; - /* for >32 bit machines */ - } -} - -/* initialize by an array with array-length */ -/* init_key is the array for initializing keys */ -/* key_length is its length */ -/* slight change for C++, 2004/2/26 */ -void mt_init_by_array(unsigned long init_key[], int key_length) -{ - int i, j, k; - mt_init_genrand(19650218UL); - i=1; j=0; - k = (N>key_length ? N : key_length); - for (; k; k--) { - mt[i] = (mt[i] ^ ((mt[i-1] ^ (mt[i-1] >> 30)) * 1664525UL)) - + init_key[j] + j; /* non linear */ - mt[i] &= 0xffffffffUL; /* for WORDSIZE > 32 machines */ - i++; j++; - if (i>=N) { mt[0] = mt[N-1]; i=1; } - if (j>=key_length) j=0; - } - for (k=N-1; k; k--) { - mt[i] = (mt[i] ^ ((mt[i-1] ^ (mt[i-1] >> 30)) * 1566083941UL)) - - i; /* non linear */ - mt[i] &= 0xffffffffUL; /* for WORDSIZE > 32 machines */ - i++; - if (i>=N) { mt[0] = mt[N-1]; i=1; } - } - - mt[0] = 0x80000000UL; /* MSB is 1; assuring non-zero initial array */ -} - -/* generates a random number on [0,0xffffffff]-interval */ -unsigned long mt_genrand_int32(void) -{ - unsigned long y; - static unsigned long mag01[2]={0x0UL, MATRIX_A}; - /* mag01[x] = x * MATRIX_A for x=0,1 */ - - if (mti >= N) { /* generate N words at one time */ - int kk; - - if (mti == N+1) /* if mt_init_genrand() has not been called, */ - mt_init_genrand(5489UL); /* a default initial seed is used */ - - for (kk=0;kk<N-M;kk++) { - y = (mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK); - mt[kk] = mt[kk+M] ^ (y >> 1) ^ mag01[y & 0x1UL]; - } - for (;kk<N-1;kk++) { - y = (mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK); - mt[kk] = mt[kk+(M-N)] ^ (y >> 1) ^ mag01[y & 0x1UL]; - } - y = (mt[N-1]&UPPER_MASK)|(mt[0]&LOWER_MASK); - mt[N-1] = mt[M-1] ^ (y >> 1) ^ mag01[y & 0x1UL]; - - mti = 0; - } - - y = mt[mti++]; - - /* Tempering */ - y ^= (y >> 11); - y ^= (y << 7) & 0x9d2c5680UL; - y ^= (y << 15) & 0xefc60000UL; - y ^= (y >> 18); - - return y; -} - -/* generates a random number on [0,0x7fffffff]-interval */ -long mt_genrand_int31(void) -{ - return (long)( mt_genrand_int32()>>1); -} - -/* generates a random number on [0,1]-real-interval */ -double mt_genrand_real1(void) -{ - return mt_genrand_int32()*(1.0/4294967295.0); - /* divided by 2^32-1 */ -} - -/* generates a random number on [0,1)-real-interval */ -double mt_genrand_real2(void) -{ - return mt_genrand_int32()*(1.0/4294967296.0); - /* divided by 2^32 */ -} - -/* generates a random number on (0,1)-real-interval */ -double mt_genrand_real3(void) -{ - return (((double) mt_genrand_int32()) + 0.5)*(1.0/4294967296.0); - /* divided by 2^32 */ -} - -/* generates a random number on [0,1) with 53-bit resolution*/ -double mt_genrand_res53(void) -{ - unsigned long a=mt_genrand_int32()>>5, b=mt_genrand_int32()>>6; - return(a*67108864.0+b)*(1.0/9007199254740992.0); -} -/* These real versions are due to Isaku Wada, 2002/01/09 added */ - -/* -#include <stdio.h> - -int main(void) -{ - int i; - unsigned long init[4]={0x123, 0x234, 0x345, 0x456}, length=4; - mt_init_by_array(init, length); - printf("1000 outputs of genrand_int32()\n"); - for (i=0; i<1000; i++) { - printf("%10lu ", mt_genrand_int32()); - if (i%5==4) printf("\n"); - } - printf("\n1000 outputs of genrand_real2()\n"); - for (i=0; i<1000; i++) { - printf("%10.8f ", mt_genrand_real2()); - if (i%5==4) printf("\n"); - } - return 0; -} -*/ diff --git a/gi/pyp-topics/src/mt19937ar.h b/gi/pyp-topics/src/mt19937ar.h deleted file mode 100644 index caab4045..00000000 --- a/gi/pyp-topics/src/mt19937ar.h +++ /dev/null @@ -1,44 +0,0 @@ -/* mt19937ar.h - * - * Mark Johnson, 17th March 2006 - */ - -#ifndef MT19937AR_H -#define MT19937AR_H - -#ifdef __cplusplus -extern "C" { -#endif - - /* initializes mt[N] with a seed */ - void mt_init_genrand(unsigned long s); - - /* initialize by an array with array-length */ - /* init_key is the array for initializing keys */ - /* key_length is its length */ - /* slight change for C++, 2004/2/26 */ - void mt_init_by_array(unsigned long init_key[], int key_length); - - /* generates a random number on [0,0xffffffff]-interval */ - unsigned long mt_genrand_int32(void); - - /* generates a random number on [0,0x7fffffff]-interval */ - long mt_genrand_int31(void); - - /* generates a random number on [0,1]-real-interval */ - double mt_genrand_real1(void); - - /* generates a random number on [0,1)-real-interval */ - double mt_genrand_real2(void); - - /* generates a random number on (0,1)-real-interval */ - double mt_genrand_real3(void); - - /* generates a random number on [0,1) with 53-bit resolution*/ - double mt_genrand_res53(void); - -#ifdef __cplusplus -}; -#endif - -#endif /* MT19937AR_H */ diff --git a/gi/pyp-topics/src/pyp-topics.cc b/gi/pyp-topics/src/pyp-topics.cc deleted file mode 100644 index 4de52fd7..00000000 --- a/gi/pyp-topics/src/pyp-topics.cc +++ /dev/null @@ -1,499 +0,0 @@ -#include "timing.h" -#include "pyp-topics.hh" -#include "contexts_corpus.hh" - -//Dict const *dict; - -//#include <boost/date_time/posix_time/posix_time_types.hpp> -void PYPTopics::sample_corpus(const Corpus& corpus, int samples, - int freq_cutoff_start, int freq_cutoff_end, - int freq_cutoff_interval, - int max_contexts_per_document, - F temp_start, F temp_end) { - Timer timer; - //dict = &((ContextsCorpus*) &corpus)->dict(); - - if (!m_backoff.get()) { - m_word_pyps.clear(); - m_word_pyps.push_back(PYPs()); - } - - std::cerr << "\n Training with " << m_word_pyps.size()-1 << " backoff level" - << (m_word_pyps.size()==2 ? ":" : "s:") << std::endl; - - - for (int i=0; i<(int)m_word_pyps.size(); ++i) - { - m_word_pyps.at(i).reserve(m_num_topics); - for (int j=0; j<m_num_topics; ++j) - m_word_pyps.at(i).push_back(new PYP<int>(0.01, 1.0, m_seed)); - } - std::cerr << std::endl; - - m_document_pyps.reserve(corpus.num_documents()); - for (int j=0; j<corpus.num_documents(); ++j) - m_document_pyps.push_back(new PYP<int>(0.01, 1.0, m_seed)); - - m_topic_p0 = 1.0/m_num_topics; - m_term_p0 = 1.0/(F)m_backoff->terms_at_level(m_word_pyps.size()-1); - //m_term_p0 = 1.0/corpus.num_types(); - m_backoff_p0 = 1.0/corpus.num_documents(); - - std::cerr << " Documents: " << corpus.num_documents() << " Terms: " - << corpus.num_types() << std::endl; - - int frequency_cutoff = freq_cutoff_start; - std::cerr << " Context frequency cutoff set to " << frequency_cutoff << std::endl; - - timer.Reset(); - // Initialisation pass - int document_id=0, topic_counter=0; - for (Corpus::const_iterator corpusIt=corpus.begin(); - corpusIt != corpus.end(); ++corpusIt, ++document_id) { - m_corpus_topics.push_back(DocumentTopics(corpusIt->size(), 0)); - - int term_index=0; - for (Document::const_iterator docIt=corpusIt->begin(); - docIt != corpusIt->end(); ++docIt, ++term_index) { - topic_counter++; - Term term = *docIt; - - // sample a new_topic - //int new_topic = (topic_counter % m_num_topics); - int freq = corpus.context_count(term); - int new_topic = -1; - if (freq > frequency_cutoff - && (!max_contexts_per_document || term_index < max_contexts_per_document)) { - //new_topic = sample(document_id, term); - //new_topic = document_id % m_num_topics; - new_topic = (int) (rnd() * m_num_topics); - - // add the new topic to the PYPs - increment(term, new_topic); - - if (m_use_topic_pyp) { - F p0 = m_topic_pyp.prob(new_topic, m_topic_p0); - int table_delta = m_document_pyps[document_id].increment(new_topic, p0); - if (table_delta) - m_topic_pyp.increment(new_topic, m_topic_p0); - } - else m_document_pyps[document_id].increment(new_topic, m_topic_p0); - } - - m_corpus_topics[document_id][term_index] = new_topic; - } - } - std::cerr << " Initialized in " << timer.Elapsed() << " seconds\n"; - - int* randomDocIndices = new int[corpus.num_documents()]; - for (int i = 0; i < corpus.num_documents(); ++i) - randomDocIndices[i] = i; - - if (num_jobs < max_threads) - num_jobs = max_threads; - int job_incr = (int) ( (float)m_document_pyps.size() / float(num_jobs) ); - - // Sampling phase - for (int curr_sample=0; curr_sample < samples; ++curr_sample) { - if (freq_cutoff_interval > 0 && curr_sample != 1 - && curr_sample % freq_cutoff_interval == 1 - && frequency_cutoff > freq_cutoff_end) { - frequency_cutoff--; - std::cerr << "\n Context frequency cutoff set to " << frequency_cutoff << std::endl; - } - - F temp = 1.0 / (temp_start - curr_sample*(temp_start-temp_end)/samples); - std::cerr << "\n -- Sample " << curr_sample << " (T=" << temp << ") "; std::cerr.flush(); - - // Randomize the corpus indexing array - int tmp; - int processed_terms=0; - /* - for (int i = corpus.num_documents()-1; i > 0; --i) - { - //i+1 since j \in [0,i] but rnd() \in [0,1) - int j = (int)(rnd() * (i+1)); - assert(j >= 0 && j <= i); - tmp = randomDocIndices[i]; - randomDocIndices[i] = randomDocIndices[j]; - randomDocIndices[j] = tmp; - } - */ - - // for each document in the corpus - int document_id; - for (int i=0; i<corpus.num_documents(); ++i) { - document_id = randomDocIndices[i]; - - // for each term in the document - int term_index=0; - Document::const_iterator docEnd = corpus.at(document_id).end(); - for (Document::const_iterator docIt=corpus.at(document_id).begin(); - docIt != docEnd; ++docIt, ++term_index) { - if (max_contexts_per_document && term_index > max_contexts_per_document) - break; - - Term term = *docIt; - - int freq = corpus.context_count(term); - if (freq < frequency_cutoff) - continue; - - processed_terms++; - - // remove the prevous topic from the PYPs - int current_topic = m_corpus_topics[document_id][term_index]; - // a negative label mean that term hasn't been sampled yet - if (current_topic >= 0) { - decrement(term, current_topic); - - int table_delta = m_document_pyps[document_id].decrement(current_topic); - if (m_use_topic_pyp && table_delta < 0) - m_topic_pyp.decrement(current_topic); - } - - // sample a new_topic - int new_topic = sample(document_id, term, temp); - //std::cerr << "TERM: " << dict->Convert(term) << " (" << term << ") " << " Old Topic: " - // << current_topic << " New Topic: " << new_topic << "\n" << std::endl; - - // add the new topic to the PYPs - m_corpus_topics[document_id][term_index] = new_topic; - increment(term, new_topic); - - if (m_use_topic_pyp) { - F p0 = m_topic_pyp.prob(new_topic, m_topic_p0); - int table_delta = m_document_pyps[document_id].increment(new_topic, p0); - if (table_delta) - m_topic_pyp.increment(new_topic, m_topic_p0); - } - else m_document_pyps[document_id].increment(new_topic, m_topic_p0); - } - if (document_id && document_id % 10000 == 0) { - std::cerr << "."; std::cerr.flush(); - } - } - std::cerr << " ||| LLH= " << log_likelihood(); - - if (curr_sample != 0 && curr_sample % 10 == 0) { - //if (true) { - std::cerr << " ||| time=" << (timer.Elapsed() / 10.0) << " sec/sample" << std::endl; - timer.Reset(); - std::cerr << " ... Resampling hyperparameters ("; - - // resample the hyperparamters - F log_p=0.0; - if (max_threads == 1) - { - std::cerr << "1 thread)" << std::endl; std::cerr.flush(); - log_p += hresample_topics(); - log_p += hresample_docs(0, m_document_pyps.size()); - } - else - { //parallelize - std::cerr << max_threads << " threads, " << num_jobs << " jobs)" << std::endl; std::cerr.flush(); - - WorkerPool<JobReturnsF, F> pool(max_threads); - int i=0, sz = m_document_pyps.size(); - //documents... - while (i <= sz - 2*job_incr) - { - JobReturnsF job = boost::bind(&PYPTopics::hresample_docs, this, i, i+job_incr); - pool.addJob(job); - i += job_incr; - } - // do all remaining documents - JobReturnsF job = boost::bind(&PYPTopics::hresample_docs, this, i,sz); - pool.addJob(job); - - //topics... - JobReturnsF topics_job = boost::bind(&PYPTopics::hresample_topics, this); - pool.addJob(topics_job); - - log_p += pool.get_result(); //blocks - - } - - if (m_use_topic_pyp) { - m_topic_pyp.resample_prior(rnd); - log_p += m_topic_pyp.log_restaurant_prob(); - } - - std::cerr.precision(10); - std::cerr << " ||| LLH=" << log_likelihood() << " ||| resampling time=" << timer.Elapsed() << " sec" << std::endl; - timer.Reset(); - - int k=0; - std::cerr << "Topics distribution: "; - std::cerr.precision(2); - for (PYPs::iterator pypIt=m_word_pyps.front().begin(); - pypIt != m_word_pyps.front().end(); ++pypIt, ++k) { - if (k % 5 == 0) std::cerr << std::endl << '\t'; - std::cerr << "<" << k << ":" << pypIt->num_customers() << "," - << pypIt->num_types() << "," << m_topic_pyp.prob(k, m_topic_p0) << "> "; - } - std::cerr.precision(10); - std::cerr << std::endl; - } - } - delete [] randomDocIndices; -} - -PYPTopics::F PYPTopics::hresample_docs(int start, int end) -{ - int resample_counter=0; - F log_p = 0.0; - assert(start >= 0); - assert(end >= 0); - assert(start <= end); - for (int i=start; i < end; ++i) - { - m_document_pyps[i].resample_prior(rnd); - log_p += m_document_pyps[i].log_restaurant_prob(); - if (resample_counter++ % 5000 == 0) { - std::cerr << "."; std::cerr.flush(); - } - } - return log_p; -} - -PYPTopics::F PYPTopics::hresample_topics() -{ - F log_p = 0.0; - for (std::vector<PYPs>::iterator levelIt=m_word_pyps.begin(); - levelIt != m_word_pyps.end(); ++levelIt) { - for (PYPs::iterator pypIt=levelIt->begin(); - pypIt != levelIt->end(); ++pypIt) { - - pypIt->resample_prior(rnd); - log_p += pypIt->log_restaurant_prob(); - } - std::cerr << log_p << std::endl; - } - return log_p; -} - -PYPTopics::F PYPTopics::log_likelihood() const -{ - F log_p = 0.0; - - // LLH of topic term distribution - size_t i=0; - for (std::vector<PYPs>::const_iterator levelIt=m_word_pyps.begin(); - levelIt != m_word_pyps.end(); ++levelIt, ++i) { - for (PYPs::const_iterator pypIt=levelIt->begin(); - pypIt != levelIt->end(); ++pypIt, ++i) { - log_p += pypIt->log_restaurant_prob(); - - if (i == m_word_pyps.size()-1) - log_p += (pypIt->num_tables() * -log(m_backoff->terms_at_level(i))); - else - log_p += (pypIt->num_tables() * log(m_term_p0)); - } - } - std::cerr << " TERM LLH: " << log_p << " "; //std::endl; - - // LLH of document topic distribution - for (size_t i=0; i < m_document_pyps.size(); ++i) { - log_p += m_document_pyps[i].log_restaurant_prob(); - if (!m_use_topic_pyp) log_p += (m_document_pyps[i].num_tables() * m_topic_p0); - } - if (m_use_topic_pyp) { - log_p += m_topic_pyp.log_restaurant_prob(); - log_p += (m_topic_pyp.num_tables() * log(m_topic_p0)); - } - - return log_p; -} - -void PYPTopics::decrement(const Term& term, int topic, int level) { - //std::cerr << "PYPTopics::decrement(" << term << "," << topic << "," << level << ")" << std::endl; - int table_delta = m_word_pyps.at(level).at(topic).decrement(term); - if (table_delta && m_backoff.get()) { - Term backoff_term = (*m_backoff)[term]; - if (!m_backoff->is_null(backoff_term)) - decrement(backoff_term, topic, level+1); - } -} - -void PYPTopics::increment(const Term& term, int topic, int level) { - //std::cerr << "PYPTopics::increment(" << term << "," << topic << "," << level << ")" << std::endl; - int table_delta = m_word_pyps.at(level).at(topic).increment(term, word_pyps_p0(term, topic, level)); - - if (table_delta && m_backoff.get()) { - Term backoff_term = (*m_backoff)[term]; - if (!m_backoff->is_null(backoff_term)) - increment(backoff_term, topic, level+1); - } -} - -int PYPTopics::sample(const DocumentId& doc, const Term& term, F inv_temp) { - // First pass: collect probs - F sum=0.0; - std::vector<F> sums; - for (int k=0; k<m_num_topics; ++k) { - F p_w_k = prob(term, k); - - F topic_prob = m_topic_p0; - if (m_use_topic_pyp) topic_prob = m_topic_pyp.prob(k, m_topic_p0); - - //F p_k_d = m_document_pyps[doc].prob(k, topic_prob); - F p_k_d = m_document_pyps[doc].unnormalised_prob(k, topic_prob); - - F prob = p_w_k*p_k_d; - /* - if (prob < 0.0) { std::cerr << "\n\n" << prob << " " << p_w_k << " " << p_k_d << std::endl; assert(false); } - if (prob > 1.0) { std::cerr << "\n\n" << prob << " " << p_w_k << " " << p_k_d << std::endl; assert(false); } - assert (pow(prob, inv_temp) >= 0.0); - assert (pow(prob, inv_temp) <= 1.0); - */ - sum += pow(prob, inv_temp); - sums.push_back(sum); - } - // Second pass: sample a topic - F cutoff = rnd() * sum; - for (int k=0; k<m_num_topics; ++k) { - if (cutoff <= sums[k]) - return k; - } - assert(false); -} - -PYPTopics::F PYPTopics::word_pyps_p0(const Term& term, int topic, int level) const { - //for (int i=0; i<level+1; ++i) std::cerr << " "; - //std::cerr << "PYPTopics::word_pyps_p0(" << term << "," << topic << "," << level << ")" << std::endl; - - F p0 = m_term_p0; - if (m_backoff.get()) { - //static F fudge=m_backoff_p0; // TODO - - Term backoff_term = (*m_backoff)[term]; - //std::cerr << "T: " << term << " BO: " << backoff_term << std::endl; - if (!m_backoff->is_null(backoff_term)) { - assert (level < m_backoff->order()); - //p0 = (1.0/(F)m_backoff->terms_at_level(level))*prob(backoff_term, topic, level+1); - p0 = m_term_p0*prob(backoff_term, topic, level+1); - p0 = prob(backoff_term, topic, level+1); - } - else - p0 = (1.0/(F) m_backoff->terms_at_level(level)); - //p0 = m_term_p0; - } - //for (int i=0; i<level+1; ++i) std::cerr << " "; - //std::cerr << "PYPTopics::word_pyps_p0(" << term << "," << topic << "," << level << ") = " << p0 << std::endl; - return p0; -} - -PYPTopics::F PYPTopics::prob(const Term& term, int topic, int level) const { - //for (int i=0; i<level+1; ++i) std::cerr << " "; - //std::cerr << "PYPTopics::prob(" << dict->Convert(term) << "," << topic << "," << level << ")" << std::endl; - - F p0 = word_pyps_p0(term, topic, level); - F p_w_k = m_word_pyps.at(level).at(topic).prob(term, p0); - - /* - for (int i=0; i<level+1; ++i) std::cerr << " "; - std::cerr << "PYPTopics::prob(" << dict->Convert(term) << "," << topic << "," << level << ") = " << p_w_k << std::endl; - for (int i=0; i<level+1; ++i) std::cerr << " "; - m_word_pyps.at(level).at(topic).debug_info(std::cerr); - */ - return p_w_k; -} - -int PYPTopics::max_topic() const { - if (!m_use_topic_pyp) - return -1; - - F current_max=0.0; - int current_topic=-1; - for (int k=0; k<m_num_topics; ++k) { - F prob = m_topic_pyp.prob(k, m_topic_p0); - if (prob > current_max) { - current_max = prob; - current_topic = k; - } - } - assert(current_topic >= 0); - return current_topic; -} - -std::pair<int,PYPTopics::F> PYPTopics::max(const DocumentId& doc) const { - //std::cerr << "PYPTopics::max(" << doc << "," << term << ")" << std::endl; - // collect probs - F current_max=0.0; - int current_topic=-1; - for (int k=0; k<m_num_topics; ++k) { - //F p_w_k = prob(term, k); - - F topic_prob = m_topic_p0; - if (m_use_topic_pyp) - topic_prob = m_topic_pyp.prob(k, m_topic_p0); - - F prob = 0; - if (doc < 0) prob = topic_prob; - else prob = m_document_pyps[doc].prob(k, topic_prob); - - if (prob > current_max) { - current_max = prob; - current_topic = k; - } - } - assert(current_topic >= 0); - assert(current_max >= 0); - return std::make_pair(current_topic, current_max); -} - -std::pair<int,PYPTopics::F> PYPTopics::max(const DocumentId& doc, const Term& term) const { - //std::cerr << "PYPTopics::max(" << doc << "," << term << ")" << std::endl; - // collect probs - F current_max=0.0; - int current_topic=-1; - for (int k=0; k<m_num_topics; ++k) { - F p_w_k = prob(term, k); - - F topic_prob = m_topic_p0; - if (m_use_topic_pyp) - topic_prob = m_topic_pyp.prob(k, m_topic_p0); - - F p_k_d = 0; - if (doc < 0) p_k_d = topic_prob; - else p_k_d = m_document_pyps[doc].prob(k, topic_prob); - - F prob = (p_w_k*p_k_d); - if (prob > current_max) { - current_max = prob; - current_topic = k; - } - } - assert(current_topic >= 0); - assert(current_max >= 0); - return std::make_pair(current_topic,current_max); -} - -std::ostream& PYPTopics::print_document_topics(std::ostream& out) const { - for (CorpusTopics::const_iterator corpusIt=m_corpus_topics.begin(); - corpusIt != m_corpus_topics.end(); ++corpusIt) { - int term_index=0; - for (DocumentTopics::const_iterator docIt=corpusIt->begin(); - docIt != corpusIt->end(); ++docIt, ++term_index) { - if (term_index) out << " "; - out << *docIt; - } - out << std::endl; - } - return out; -} - -std::ostream& PYPTopics::print_topic_terms(std::ostream& out) const { - for (PYPs::const_iterator pypsIt=m_word_pyps.front().begin(); - pypsIt != m_word_pyps.front().end(); ++pypsIt) { - int term_index=0; - for (PYP<int>::const_iterator termIt=pypsIt->begin(); - termIt != pypsIt->end(); ++termIt, ++term_index) { - if (term_index) out << " "; - out << termIt->first << ":" << termIt->second; - } - out << std::endl; - } - return out; -} diff --git a/gi/pyp-topics/src/pyp-topics.hh b/gi/pyp-topics/src/pyp-topics.hh deleted file mode 100644 index 3a910540..00000000 --- a/gi/pyp-topics/src/pyp-topics.hh +++ /dev/null @@ -1,98 +0,0 @@ -#ifndef PYP_TOPICS_HH -#define PYP_TOPICS_HH - -#include <vector> -#include <iostream> -#include <boost/ptr_container/ptr_vector.hpp> - -#include <boost/random/uniform_real.hpp> -#include <boost/random/variate_generator.hpp> -#include <boost/random/mersenne_twister.hpp> - -#include "pyp.hh" -#include "corpus.hh" -#include "workers.hh" - -class PYPTopics { -public: - typedef std::vector<int> DocumentTopics; - typedef std::vector<DocumentTopics> CorpusTopics; - typedef long double F; - -public: - PYPTopics(int num_topics, bool use_topic_pyp=false, unsigned long seed = 0, - int max_threads = 1, int num_jobs = 1) - : m_num_topics(num_topics), m_word_pyps(1), - m_topic_pyp(0.5,1.0,seed), m_use_topic_pyp(use_topic_pyp), - m_seed(seed), - uni_dist(0,1), rng(seed == 0 ? (unsigned long)this : seed), - rnd(rng, uni_dist), max_threads(max_threads), num_jobs(num_jobs) {} - - void sample_corpus(const Corpus& corpus, int samples, - int freq_cutoff_start=0, int freq_cutoff_end=0, - int freq_cutoff_interval=0, - int max_contexts_per_document=0, - F temp_start=1.0, F temp_end=1.0); - - int sample(const DocumentId& doc, const Term& term, F inv_temp=1.0); - std::pair<int,F> max(const DocumentId& doc, const Term& term) const; - std::pair<int,F> max(const DocumentId& doc) const; - int max_topic() const; - - void set_backoff(const std::string& filename) { - m_backoff.reset(new TermBackoff); - m_backoff->read(filename); - m_word_pyps.clear(); - m_word_pyps.resize(m_backoff->order(), PYPs()); - } - void set_backoff(TermBackoffPtr backoff) { - m_backoff = backoff; - m_word_pyps.clear(); - m_word_pyps.resize(m_backoff->order(), PYPs()); - } - - F prob(const Term& term, int topic, int level=0) const; - void decrement(const Term& term, int topic, int level=0); - void increment(const Term& term, int topic, int level=0); - - F log_likelihood() const; - - std::ostream& print_document_topics(std::ostream& out) const; - std::ostream& print_topic_terms(std::ostream& out) const; - -private: - F word_pyps_p0(const Term& term, int topic, int level) const; - - int m_num_topics; - F m_term_p0, m_topic_p0, m_backoff_p0; - - CorpusTopics m_corpus_topics; - typedef boost::ptr_vector< PYP<int> > PYPs; - PYPs m_document_pyps; - std::vector<PYPs> m_word_pyps; - PYP<int> m_topic_pyp; - bool m_use_topic_pyp; - - unsigned long m_seed; - - typedef boost::mt19937 base_generator_type; - typedef boost::uniform_real<> uni_dist_type; - typedef boost::variate_generator<base_generator_type&, uni_dist_type> gen_type; - - uni_dist_type uni_dist; - base_generator_type rng; //this gets the seed - gen_type rnd; //instantiate: rnd(rng, uni_dist) - //call: rnd() generates uniform on [0,1) - - typedef boost::function<F()> JobReturnsF; - - F hresample_docs(int start, int end); //does i in [start, end) - - F hresample_topics(); - - int max_threads; - int num_jobs; - TermBackoffPtr m_backoff; -}; - -#endif // PYP_TOPICS_HH diff --git a/gi/pyp-topics/src/pyp.hh b/gi/pyp-topics/src/pyp.hh deleted file mode 100644 index b1cb62be..00000000 --- a/gi/pyp-topics/src/pyp.hh +++ /dev/null @@ -1,566 +0,0 @@ -#ifndef _pyp_hh -#define _pyp_hh - -#include "slice-sampler.h" -#include <math.h> -#include <map> -#include <tr1/unordered_map> -//#include <google/sparse_hash_map> - -#include <boost/random/uniform_real.hpp> -#include <boost/random/variate_generator.hpp> -#include <boost/random/mersenne_twister.hpp> - -#include "log_add.h" -#include "mt19937ar.h" - -// -// Pitman-Yor process with customer and table tracking -// - -template <typename Dish, typename Hash=std::tr1::hash<Dish> > -class PYP : protected std::tr1::unordered_map<Dish, int, Hash> -//class PYP : protected google::sparse_hash_map<Dish, int, Hash> -{ -public: - using std::tr1::unordered_map<Dish,int>::const_iterator; - using std::tr1::unordered_map<Dish,int>::iterator; - using std::tr1::unordered_map<Dish,int>::begin; - using std::tr1::unordered_map<Dish,int>::end; -// using google::sparse_hash_map<Dish,int>::const_iterator; -// using google::sparse_hash_map<Dish,int>::iterator; -// using google::sparse_hash_map<Dish,int>::begin; -// using google::sparse_hash_map<Dish,int>::end; - - PYP(double a, double b, unsigned long seed = 0, Hash hash=Hash()); - - virtual int increment(Dish d, double p0); - virtual int decrement(Dish d); - - // lookup functions - int count(Dish d) const; - double prob(Dish dish, double p0) const; - double prob(Dish dish, double dcd, double dca, - double dtd, double dta, double p0) const; - double unnormalised_prob(Dish dish, double p0) const; - - int num_customers() const { return _total_customers; } - int num_types() const { return std::tr1::unordered_map<Dish,int>::size(); } - //int num_types() const { return google::sparse_hash_map<Dish,int>::size(); } - bool empty() const { return _total_customers == 0; } - - double log_prob(Dish dish, double log_p0) const; - // nb. d* are NOT logs - double log_prob(Dish dish, double dcd, double dca, - double dtd, double dta, double log_p0) const; - - int num_tables(Dish dish) const; - int num_tables() const; - - double a() const { return _a; } - void set_a(double a) { _a = a; } - - double b() const { return _b; } - void set_b(double b) { _b = b; } - - virtual void clear(); - std::ostream& debug_info(std::ostream& os) const; - - double log_restaurant_prob() const; - double log_prior() const; - static double log_prior_a(double a, double beta_a, double beta_b); - static double log_prior_b(double b, double gamma_c, double gamma_s); - - template <typename Uniform01> - void resample_prior(Uniform01& rnd); - template <typename Uniform01> - void resample_prior_a(Uniform01& rnd); - template <typename Uniform01> - void resample_prior_b(Uniform01& rnd); - -protected: - double _a, _b; // parameters of the Pitman-Yor distribution - double _a_beta_a, _a_beta_b; // parameters of Beta prior on a - double _b_gamma_s, _b_gamma_c; // parameters of Gamma prior on b - - struct TableCounter { - TableCounter() : tables(0) {}; - int tables; - std::map<int, int> table_histogram; // num customers at table -> number tables - }; - typedef std::tr1::unordered_map<Dish, TableCounter, Hash> DishTableType; - //typedef google::sparse_hash_map<Dish, TableCounter, Hash> DishTableType; - DishTableType _dish_tables; - int _total_customers, _total_tables; - - typedef boost::mt19937 base_generator_type; - typedef boost::uniform_real<> uni_dist_type; - typedef boost::variate_generator<base_generator_type&, uni_dist_type> gen_type; - -// uni_dist_type uni_dist; -// base_generator_type rng; //this gets the seed -// gen_type rnd; //instantiate: rnd(rng, uni_dist) - //call: rnd() generates uniform on [0,1) - - // Function objects for calculating the parts of the log_prob for - // the parameters a and b - struct resample_a_type { - int n, m; double b, a_beta_a, a_beta_b; - const DishTableType& dish_tables; - resample_a_type(int n, int m, double b, double a_beta_a, - double a_beta_b, const DishTableType& dish_tables) - : n(n), m(m), b(b), a_beta_a(a_beta_a), a_beta_b(a_beta_b), dish_tables(dish_tables) {} - - double operator() (double proposed_a) const { - double log_prior = log_prior_a(proposed_a, a_beta_a, a_beta_b); - double log_prob = 0.0; - double lgamma1a = lgamma(1.0 - proposed_a); - for (typename DishTableType::const_iterator dish_it=dish_tables.begin(); dish_it != dish_tables.end(); ++dish_it) - for (std::map<int, int>::const_iterator table_it=dish_it->second.table_histogram.begin(); - table_it !=dish_it->second.table_histogram.end(); ++table_it) - log_prob += (table_it->second * (lgamma(table_it->first - proposed_a) - lgamma1a)); - - log_prob += (proposed_a == 0.0 ? (m-1.0)*log(b) - : ((m-1.0)*log(proposed_a) + lgamma((m-1.0) + b/proposed_a) - lgamma(b/proposed_a))); - assert(std::isfinite(log_prob)); - return log_prob + log_prior; - } - }; - - struct resample_b_type { - int n, m; double a, b_gamma_c, b_gamma_s; - resample_b_type(int n, int m, double a, double b_gamma_c, double b_gamma_s) - : n(n), m(m), a(a), b_gamma_c(b_gamma_c), b_gamma_s(b_gamma_s) {} - - double operator() (double proposed_b) const { - double log_prior = log_prior_b(proposed_b, b_gamma_c, b_gamma_s); - double log_prob = 0.0; - log_prob += (a == 0.0 ? (m-1.0)*log(proposed_b) - : ((m-1.0)*log(a) + lgamma((m-1.0) + proposed_b/a) - lgamma(proposed_b/a))); - log_prob += (lgamma(1.0+proposed_b) - lgamma(n+proposed_b)); - return log_prob + log_prior; - } - }; - - /* lbetadist() returns the log probability density of x under a Beta(alpha,beta) - * distribution. - copied from Mark Johnson's gammadist.c - */ - static long double lbetadist(long double x, long double alpha, long double beta); - - /* lgammadist() returns the log probability density of x under a Gamma(alpha,beta) - * distribution - copied from Mark Johnson's gammadist.c - */ - static long double lgammadist(long double x, long double alpha, long double beta); - -}; - -template <typename Dish, typename Hash> -PYP<Dish,Hash>::PYP(double a, double b, unsigned long seed, Hash) -: std::tr1::unordered_map<Dish, int, Hash>(10), _a(a), _b(b), -//: google::sparse_hash_map<Dish, int, Hash>(10), _a(a), _b(b), - _a_beta_a(1), _a_beta_b(1), _b_gamma_s(1), _b_gamma_c(1), - //_a_beta_a(1), _a_beta_b(1), _b_gamma_s(10), _b_gamma_c(0.1), - _total_customers(0), _total_tables(0)//, - //uni_dist(0,1), rng(seed == 0 ? (unsigned long)this : seed), rnd(rng, uni_dist) -{ -// std::cerr << "\t##PYP<Dish,Hash>::PYP(a=" << _a << ",b=" << _b << ")" << std::endl; - //set_deleted_key(-std::numeric_limits<Dish>::max()); -} - -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::prob(Dish dish, double p0) const -{ - int c = count(dish), t = num_tables(dish); - double r = num_tables() * _a + _b; - //std::cerr << "\t\t\t\tPYP<Dish,Hash>::prob(" << dish << "," << p0 << ") c=" << c << " r=" << r << std::endl; - if (c > 0) - return (c - _a * t + r * p0) / (num_customers() + _b); - else - return r * p0 / (num_customers() + _b); -} - -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::unnormalised_prob(Dish dish, double p0) const -{ - int c = count(dish), t = num_tables(dish); - double r = num_tables() * _a + _b; - if (c > 0) return (c - _a * t + r * p0); - else return r * p0; -} - -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::prob(Dish dish, double dcd, double dca, - double dtd, double dta, double p0) -const -{ - int c = count(dish) + dcd, t = num_tables(dish) + dtd; - double r = (num_tables() + dta) * _a + _b; - if (c > 0) - return (c - _a * t + r * p0) / (num_customers() + dca + _b); - else - return r * p0 / (num_customers() + dca + _b); -} - -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::log_prob(Dish dish, double log_p0) const -{ - using std::log; - int c = count(dish), t = num_tables(dish); - double r = log(num_tables() * _a + b); - if (c > 0) - return Log<double>::add(log(c - _a * t), r + log_p0) - - log(num_customers() + _b); - else - return r + log_p0 - log(num_customers() + b); -} - -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::log_prob(Dish dish, double dcd, double dca, - double dtd, double dta, double log_p0) -const -{ - using std::log; - int c = count(dish) + dcd, t = num_tables(dish) + dtd; - double r = log((num_tables() + dta) * _a + b); - if (c > 0) - return Log<double>::add(log(c - _a * t), r + log_p0) - - log(num_customers() + dca + _b); - else - return r + log_p0 - log(num_customers() + dca + b); -} - -template <typename Dish, typename Hash> -int -PYP<Dish,Hash>::increment(Dish dish, double p0) { - int delta = 0; - TableCounter &tc = _dish_tables[dish]; - - // seated on a new or existing table? - int c = count(dish), t = num_tables(dish), T = num_tables(); - double pshare = (c > 0) ? (c - _a*t) : 0.0; - double pnew = (_b + _a*T) * p0; - assert (pshare >= 0.0); - //assert (pnew > 0.0); - - //if (rnd() < pnew / (pshare + pnew)) { - if (mt_genrand_res53() < pnew / (pshare + pnew)) { - // assign to a new table - tc.tables += 1; - tc.table_histogram[1] += 1; - _total_tables += 1; - delta = 1; - } - else { - // randomly assign to an existing table - // remove constant denominator from inner loop - //double r = rnd() * (c - _a*t); - double r = mt_genrand_res53() * (c - _a*t); - for (std::map<int,int>::iterator - hit = tc.table_histogram.begin(); - hit != tc.table_histogram.end(); ++hit) { - r -= ((hit->first - _a) * hit->second); - if (r <= 0) { - tc.table_histogram[hit->first+1] += 1; - hit->second -= 1; - if (hit->second == 0) - tc.table_histogram.erase(hit); - break; - } - } - if (r > 0) { - std::cerr << r << " " << c << " " << _a << " " << t << std::endl; - assert(false); - } - delta = 0; - } - - std::tr1::unordered_map<Dish,int,Hash>::operator[](dish) += 1; - //google::sparse_hash_map<Dish,int,Hash>::operator[](dish) += 1; - _total_customers += 1; - - return delta; -} - -template <typename Dish, typename Hash> -int -PYP<Dish,Hash>::count(Dish dish) const -{ - typename std::tr1::unordered_map<Dish, int>::const_iterator - //typename google::sparse_hash_map<Dish, int>::const_iterator - dcit = find(dish); - if (dcit != end()) - return dcit->second; - else - return 0; -} - -template <typename Dish, typename Hash> -int -PYP<Dish,Hash>::decrement(Dish dish) -{ - typename std::tr1::unordered_map<Dish, int>::iterator dcit = find(dish); - //typename google::sparse_hash_map<Dish, int>::iterator dcit = find(dish); - if (dcit == end()) { - std::cerr << dish << std::endl; - assert(false); - } - - int delta = 0; - - typename std::tr1::unordered_map<Dish, TableCounter>::iterator dtit = _dish_tables.find(dish); - //typename google::sparse_hash_map<Dish, TableCounter>::iterator dtit = _dish_tables.find(dish); - if (dtit == _dish_tables.end()) { - std::cerr << dish << std::endl; - assert(false); - } - TableCounter &tc = dtit->second; - - //std::cerr << "\tdecrement for " << dish << "\n"; - //std::cerr << "\tBEFORE histogram: " << tc.table_histogram << " "; - //std::cerr << "count: " << count(dish) << " "; - //std::cerr << "tables: " << tc.tables << "\n"; - - //double r = rnd() * count(dish); - double r = mt_genrand_res53() * count(dish); - for (std::map<int,int>::iterator hit = tc.table_histogram.begin(); - hit != tc.table_histogram.end(); ++hit) - { - //r -= (hit->first - _a) * hit->second; - r -= (hit->first) * hit->second; - if (r <= 0) - { - if (hit->first > 1) - tc.table_histogram[hit->first-1] += 1; - else - { - delta = -1; - tc.tables -= 1; - _total_tables -= 1; - } - - hit->second -= 1; - if (hit->second == 0) tc.table_histogram.erase(hit); - break; - } - } - if (r > 0) { - std::cerr << r << " " << count(dish) << " " << _a << " " << num_tables(dish) << std::endl; - assert(false); - } - - // remove the customer - dcit->second -= 1; - _total_customers -= 1; - assert(dcit->second >= 0); - if (dcit->second == 0) { - erase(dcit); - _dish_tables.erase(dtit); - //std::cerr << "\tAFTER histogram: Empty\n"; - } - else { - //std::cerr << "\tAFTER histogram: " << _dish_tables[dish].table_histogram << " "; - //std::cerr << "count: " << count(dish) << " "; - //std::cerr << "tables: " << _dish_tables[dish].tables << "\n"; - } - - return delta; -} - -template <typename Dish, typename Hash> -int -PYP<Dish,Hash>::num_tables(Dish dish) const -{ - typename std::tr1::unordered_map<Dish, TableCounter, Hash>::const_iterator - //typename google::sparse_hash_map<Dish, TableCounter, Hash>::const_iterator - dtit = _dish_tables.find(dish); - - //assert(dtit != _dish_tables.end()); - if (dtit == _dish_tables.end()) - return 0; - - return dtit->second.tables; -} - -template <typename Dish, typename Hash> -int -PYP<Dish,Hash>::num_tables() const -{ - return _total_tables; -} - -template <typename Dish, typename Hash> -std::ostream& -PYP<Dish,Hash>::debug_info(std::ostream& os) const -{ - int hists = 0, tables = 0; - for (typename std::tr1::unordered_map<Dish, TableCounter, Hash>::const_iterator - //for (typename google::sparse_hash_map<Dish, TableCounter, Hash>::const_iterator - dtit = _dish_tables.begin(); dtit != _dish_tables.end(); ++dtit) - { - hists += dtit->second.table_histogram.size(); - tables += dtit->second.tables; - -// if (dtit->second.tables <= 0) -// std::cerr << dtit->first << " " << count(dtit->first) << std::endl; - assert(dtit->second.tables > 0); - assert(!dtit->second.table_histogram.empty()); - -// os << "Dish " << dtit->first << " has " << count(dtit->first) << " customers, and is sitting at " << dtit->second.tables << " tables.\n"; - for (std::map<int,int>::const_iterator - hit = dtit->second.table_histogram.begin(); - hit != dtit->second.table_histogram.end(); ++hit) { -// os << " " << hit->second << " tables with " << hit->first << " customers." << std::endl; - assert(hit->second > 0); - } - } - - os << "restaurant has " - << _total_customers << " customers; " - << _total_tables << " tables; " - << tables << " tables'; " - << num_types() << " dishes; " - << _dish_tables.size() << " dishes'; and " - << hists << " histogram entries\n"; - - return os; -} - -template <typename Dish, typename Hash> -void -PYP<Dish,Hash>::clear() -{ - this->std::tr1::unordered_map<Dish,int,Hash>::clear(); - //this->google::sparse_hash_map<Dish,int,Hash>::clear(); - _dish_tables.clear(); - _total_tables = _total_customers = 0; -} - -// log_restaurant_prob returns the log probability of the PYP table configuration. -// Excludes Hierarchical P0 term which must be calculated separately. -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::log_restaurant_prob() const { - if (_total_customers < 1) - return (double)0.0; - - double log_prob = 0.0; - double lgamma1a = lgamma(1.0-_a); - - //std::cerr << "-------------------\n" << std::endl; - for (typename DishTableType::const_iterator dish_it=_dish_tables.begin(); - dish_it != _dish_tables.end(); ++dish_it) { - for (std::map<int, int>::const_iterator table_it=dish_it->second.table_histogram.begin(); - table_it !=dish_it->second.table_histogram.end(); ++table_it) { - log_prob += (table_it->second * (lgamma(table_it->first - _a) - lgamma1a)); - //std::cerr << "|" << dish_it->first->parent << " --> " << dish_it->first->rhs << " " << table_it->first << " " << table_it->second << " " << log_prob; - } - } - //std::cerr << std::endl; - - log_prob += (_a == (double)0.0 ? (_total_tables-1.0)*log(_b) : (_total_tables-1.0)*log(_a) + lgamma((_total_tables-1.0) + _b/_a) - lgamma(_b/_a)); - //std::cerr << "\t\t" << log_prob << std::endl; - log_prob += (lgamma(1.0 + _b) - lgamma(_total_customers + _b)); - - //std::cerr << _total_customers << " " << _total_tables << " " << log_prob << " " << log_prior() << std::endl; - //std::cerr << _a << " " << _b << std::endl; - if (!std::isfinite(log_prob)) { - assert(false); - } - //return log_prob; - if (log_prob > 0.0) - std::cerr << log_prob << std::endl; - return log_prob;// + log_prior(); -} - -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::log_prior() const { - double prior = 0.0; - if (_a_beta_a > 0.0 && _a_beta_b > 0.0 && _a > 0.0) - prior += log_prior_a(_a, _a_beta_a, _a_beta_b); - if (_b_gamma_s > 0.0 && _b_gamma_c > 0.0) - prior += log_prior_b(_b, _b_gamma_c, _b_gamma_s); - - return prior; -} - -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::log_prior_a(double a, double beta_a, double beta_b) { - return lbetadist(a, beta_a, beta_b); -} - -template <typename Dish, typename Hash> -double -PYP<Dish,Hash>::log_prior_b(double b, double gamma_c, double gamma_s) { - return lgammadist(b, gamma_c, gamma_s); -} - -template <typename Dish, typename Hash> -long double PYP<Dish,Hash>::lbetadist(long double x, long double alpha, long double beta) { - assert(x > 0); - assert(x < 1); - assert(alpha > 0); - assert(beta > 0); - return (alpha-1)*log(x)+(beta-1)*log(1-x)+lgamma(alpha+beta)-lgamma(alpha)-lgamma(beta); -//boost::math::lgamma -} - -template <typename Dish, typename Hash> -long double PYP<Dish,Hash>::lgammadist(long double x, long double alpha, long double beta) { - assert(alpha > 0); - assert(beta > 0); - return (alpha-1)*log(x) - alpha*log(beta) - x/beta - lgamma(alpha); -} - - -template <typename Dish, typename Hash> - template <typename Uniform01> -void -PYP<Dish,Hash>::resample_prior(Uniform01& rnd) { - for (int num_its=5; num_its >= 0; --num_its) { - resample_prior_b(rnd); - resample_prior_a(rnd); - } - resample_prior_b(rnd); -} - -template <typename Dish, typename Hash> - template <typename Uniform01> -void -PYP<Dish,Hash>::resample_prior_b(Uniform01& rnd) { - if (_total_tables == 0) - return; - - //int niterations = 10; // number of resampling iterations - int niterations = 5; // number of resampling iterations - //std::cerr << "\n## resample_prior_b(), initial a = " << _a << ", b = " << _b << std::endl; - resample_b_type b_log_prob(_total_customers, _total_tables, _a, _b_gamma_c, _b_gamma_s); - _b = slice_sampler1d(b_log_prob, _b, rnd, (double) 0.0, std::numeric_limits<double>::infinity(), - //_b = slice_sampler1d(b_log_prob, _b, mt_genrand_res53, (double) 0.0, std::numeric_limits<double>::infinity(), - (double) 0.0, niterations, 100*niterations); - //std::cerr << "\n## resample_prior_b(), final a = " << _a << ", b = " << _b << std::endl; -} - -template <typename Dish, typename Hash> - template <typename Uniform01> -void -PYP<Dish,Hash>::resample_prior_a(Uniform01& rnd) { - if (_total_tables == 0) - return; - - //int niterations = 10; - int niterations = 5; - //std::cerr << "\n## Initial a = " << _a << ", b = " << _b << std::endl; - resample_a_type a_log_prob(_total_customers, _total_tables, _b, _a_beta_a, _a_beta_b, _dish_tables); - _a = slice_sampler1d(a_log_prob, _a, rnd, std::numeric_limits<double>::min(), - //_a = slice_sampler1d(a_log_prob, _a, mt_genrand_res53, std::numeric_limits<double>::min(), - (double) 1.0, (double) 0.0, niterations, 100*niterations); -} - -#endif diff --git a/gi/pyp-topics/src/slice-sampler.h b/gi/pyp-topics/src/slice-sampler.h deleted file mode 100644 index 3108a0f7..00000000 --- a/gi/pyp-topics/src/slice-sampler.h +++ /dev/null @@ -1,192 +0,0 @@ -//! slice-sampler.h is an MCMC slice sampler -//! -//! Mark Johnson, 1st August 2008 - -#ifndef SLICE_SAMPLER_H -#define SLICE_SAMPLER_H - -#include <algorithm> -#include <cassert> -#include <cmath> -#include <iostream> -#include <limits> - -//! slice_sampler_rfc_type{} returns the value of a user-specified -//! function if the argument is within range, or - infinity otherwise -// -template <typename F, typename Fn, typename U> -struct slice_sampler_rfc_type { - F min_x, max_x; - const Fn& f; - U max_nfeval, nfeval; - slice_sampler_rfc_type(F min_x, F max_x, const Fn& f, U max_nfeval) - : min_x(min_x), max_x(max_x), f(f), max_nfeval(max_nfeval), nfeval(0) { } - - F operator() (F x) { - if (min_x < x && x < max_x) { - assert(++nfeval <= max_nfeval); - F fx = f(x); - assert(std::isfinite(fx)); - return fx; - } - else - return -std::numeric_limits<F>::infinity(); - } -}; // slice_sampler_rfc_type{} - -//! slice_sampler1d() implements the univariate "range doubling" slice sampler -//! described in Neal (2003) "Slice Sampling", The Annals of Statistics 31(3), 705-767. -// -template <typename F, typename LogF, typename Uniform01> -F slice_sampler1d(const LogF& logF0, //!< log of function to sample - F x, //!< starting point - Uniform01& u01, //!< uniform [0,1) random number generator - F min_x = -std::numeric_limits<F>::infinity(), //!< minimum value of support - F max_x = std::numeric_limits<F>::infinity(), //!< maximum value of support - F w = 0.0, //!< guess at initial width - unsigned nsamples=1, //!< number of samples to draw - unsigned max_nfeval=200) //!< max number of function evaluations -{ - typedef unsigned U; - slice_sampler_rfc_type<F,LogF,U> logF(min_x, max_x, logF0, max_nfeval); - - assert(std::isfinite(x)); - - if (w <= 0.0) { // set w to a default width - if (min_x > -std::numeric_limits<F>::infinity() && max_x < std::numeric_limits<F>::infinity()) - w = (max_x - min_x)/4; - else - w = std::max(((x < 0.0) ? -x : x)/4, (F) 0.1); - } - assert(std::isfinite(w)); - - F logFx = logF(x); - for (U sample = 0; sample < nsamples; ++sample) { - F logY = logFx + log(u01()+1e-100); //! slice logFx at this value - assert(std::isfinite(logY)); - - F xl = x - w*u01(); //! lower bound on slice interval - F logFxl = logF(xl); - F xr = xl + w; //! upper bound on slice interval - F logFxr = logF(xr); - - while (logY < logFxl || logY < logFxr) // doubling procedure - if (u01() < 0.5) - logFxl = logF(xl -= xr - xl); - else - logFxr = logF(xr += xr - xl); - - F xl1 = xl; - F xr1 = xr; - while (true) { // shrinking procedure - F x1 = xl1 + u01()*(xr1 - xl1); - if (logY < logF(x1)) { - F xl2 = xl; // acceptance procedure - F xr2 = xr; - bool d = false; - while (xr2 - xl2 > 1.1*w) { - F xm = (xl2 + xr2)/2; - if ((x < xm && x1 >= xm) || (x >= xm && x1 < xm)) - d = true; - if (x1 < xm) - xr2 = xm; - else - xl2 = xm; - if (d && logY >= logF(xl2) && logY >= logF(xr2)) - goto unacceptable; - } - x = x1; - goto acceptable; - } - goto acceptable; - unacceptable: - if (x1 < x) // rest of shrinking procedure - xl1 = x1; - else - xr1 = x1; - } - acceptable: - w = (4*w + (xr1 - xl1))/5; // update width estimate - } - return x; -} - -/* -//! slice_sampler1d() implements a 1-d MCMC slice sampler. -//! It should be correct for unimodal distributions, but -//! not for multimodal ones. -// -template <typename F, typename LogP, typename Uniform01> -F slice_sampler1d(const LogP& logP, //!< log of distribution to sample - F x, //!< initial sample - Uniform01& u01, //!< uniform random number generator - F min_x = -std::numeric_limits<F>::infinity(), //!< minimum value of support - F max_x = std::numeric_limits<F>::infinity(), //!< maximum value of support - F w = 0.0, //!< guess at initial width - unsigned nsamples=1, //!< number of samples to draw - unsigned max_nfeval=200) //!< max number of function evaluations -{ - typedef unsigned U; - assert(std::isfinite(x)); - if (w <= 0.0) { - if (min_x > -std::numeric_limits<F>::infinity() && max_x < std::numeric_limits<F>::infinity()) - w = (max_x - min_x)/4; - else - w = std::max(((x < 0.0) ? -x : x)/4, 0.1); - } - // TRACE4(x, min_x, max_x, w); - F logPx = logP(x); - assert(std::isfinite(logPx)); - U nfeval = 1; - for (U sample = 0; sample < nsamples; ++sample) { - F x0 = x; - F logU = logPx + log(u01()+1e-100); - assert(std::isfinite(logU)); - F r = u01(); - F xl = std::max(min_x, x - r*w); - F xr = std::min(max_x, x + (1-r)*w); - // TRACE3(x, logPx, logU); - while (xl > min_x && logP(xl) > logU) { - xl -= w; - w *= 2; - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xl = " << xl << std::endl; - assert(nfeval < max_nfeval); - } - xl = std::max(xl, min_x); - while (xr < max_x && logP(xr) > logU) { - xr += w; - w *= 2; - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xr = " << xr << std::endl; - assert(nfeval < max_nfeval); - } - xr = std::min(xr, max_x); - while (true) { - r = u01(); - x = r*xl + (1-r)*xr; - assert(std::isfinite(x)); - logPx = logP(x); - // TRACE4(logPx, x, xl, xr); - assert(std::isfinite(logPx)); - ++nfeval; - if (nfeval >= max_nfeval) - std::cerr << "## Error: nfeval = " << nfeval << ", max_nfeval = " << max_nfeval << ", sample = " << sample << ", nsamples = " << nsamples << ", r = " << r << ", w = " << w << ", xl = " << xl << ", xr = " << xr << ", x = " << x << std::endl; - assert(nfeval < max_nfeval); - if (logPx > logU) - break; - else if (x > x0) - xr = x; - else - xl = x; - } - // w = (4*w + (xr-xl))/5; // gradually adjust w - } - // TRACE2(logPx, x); - return x; -} // slice_sampler1d() -*/ - -#endif // SLICE_SAMPLER_H diff --git a/gi/pyp-topics/src/timing.h b/gi/pyp-topics/src/timing.h deleted file mode 100644 index 08360b0f..00000000 --- a/gi/pyp-topics/src/timing.h +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef TIMING_H -#define TIMING_H - -#ifdef __CYGWIN__ -# ifndef _POSIX_MONOTONIC_CLOCK -# define _POSIX_MONOTONIC_CLOCK -// this modifies <time.h> -# endif -// in case someone included <time.h> before we got here (this is lifted from time.h>) -# ifndef CLOCK_MONOTONIC -# define CLOCK_MONOTONIC (clockid_t)4 -# endif -#endif - - -#include <time.h> -#include <sys/time.h> -#include "clock_gettime_stub.c" - -struct Timer { - Timer() { Reset(); } - void Reset() - { - clock_gettime(CLOCK_MONOTONIC, &start_t); - } - double Elapsed() const { - timespec end_t; - clock_gettime(CLOCK_MONOTONIC, &end_t); - const double elapsed = (end_t.tv_sec - start_t.tv_sec) - + (end_t.tv_nsec - start_t.tv_nsec) / 1000000000.0; - return elapsed; - } - private: - timespec start_t; -}; - -#endif diff --git a/gi/pyp-topics/src/train-contexts.cc b/gi/pyp-topics/src/train-contexts.cc deleted file mode 100644 index 9463f9fc..00000000 --- a/gi/pyp-topics/src/train-contexts.cc +++ /dev/null @@ -1,174 +0,0 @@ -// STL -#include <iostream> -#include <fstream> -#include <algorithm> -#include <iterator> - -// Boost -#include <boost/program_options/parsers.hpp> -#include <boost/program_options/variables_map.hpp> -#include <boost/scoped_ptr.hpp> - -// Local -#include "pyp-topics.hh" -#include "corpus.hh" -#include "contexts_corpus.hh" -#include "gzstream.hh" - -static const char *REVISION = "$Rev$"; - -// Namespaces -using namespace boost; -using namespace boost::program_options; -using namespace std; - -int main(int argc, char **argv) -{ - cout << "Pitman Yor topic models: Copyright 2010 Phil Blunsom\n"; - cout << REVISION << '\n' <<endl; - - //////////////////////////////////////////////////////////////////////////////////////////// - // Command line processing - variables_map vm; - - // Command line processing - { - options_description cmdline_specific("Command line specific options"); - cmdline_specific.add_options() - ("help,h", "print help message") - ("config,c", value<string>(), "config file specifying additional command line options") - ; - options_description config_options("Allowed options"); - config_options.add_options() - ("data,d", value<string>(), "file containing the documents and context terms") - ("topics,t", value<int>()->default_value(50), "number of topics") - ("document-topics-out,o", value<string>(), "file to write the document topics to") - ("default-topics-out", value<string>(), "file to write default term topic assignments.") - ("topic-words-out,w", value<string>(), "file to write the topic word distribution to") - ("samples,s", value<int>()->default_value(10), "number of sampling passes through the data") - ("backoff-type", value<string>(), "backoff type: none|simple") -// ("filter-singleton-contexts", "filter singleton contexts") - ("hierarchical-topics", "Use a backoff hierarchical PYP as the P0 for the document topics distribution.") - ("freq-cutoff-start", value<int>()->default_value(0), "initial frequency cutoff.") - ("freq-cutoff-end", value<int>()->default_value(0), "final frequency cutoff.") - ("freq-cutoff-interval", value<int>()->default_value(0), "number of iterations between frequency decrement.") - ("max-threads", value<int>()->default_value(1), "maximum number of simultaneous threads allowed") - ("max-contexts-per-document", value<int>()->default_value(0), "Only sample the n most frequent contexts for a document.") - ("num-jobs", value<int>()->default_value(1), "allows finer control over parallelization") - ("temp-start", value<double>()->default_value(1.0), "starting annealing temperature.") - ("temp-end", value<double>()->default_value(1.0), "end annealing temperature.") - ; - - cmdline_specific.add(config_options); - - store(parse_command_line(argc, argv, cmdline_specific), vm); - notify(vm); - - if (vm.count("config") > 0) { - ifstream config(vm["config"].as<string>().c_str()); - store(parse_config_file(config, config_options), vm); - } - - if (vm.count("help")) { - cout << cmdline_specific << "\n"; - return 1; - } - } - //////////////////////////////////////////////////////////////////////////////////////////// - - if (!vm.count("data")) { - cerr << "Please specify a file containing the data." << endl; - return 1; - } - assert(vm["max-threads"].as<int>() > 0); - assert(vm["num-jobs"].as<int>() > -1); - // seed the random number generator: 0 = automatic, specify value otherwise - unsigned long seed = 0; - PYPTopics model(vm["topics"].as<int>(), vm.count("hierarchical-topics"), seed, vm["max-threads"].as<int>(), vm["num-jobs"].as<int>()); - - // read the data - BackoffGenerator* backoff_gen=0; - if (vm.count("backoff-type")) { - if (vm["backoff-type"].as<std::string>() == "none") { - backoff_gen = 0; - } - else if (vm["backoff-type"].as<std::string>() == "simple") { - backoff_gen = new SimpleBackoffGenerator(); - } - else { - cerr << "Backoff type (--backoff-type) must be one of none|simple." <<endl; - return(1); - } - } - - ContextsCorpus contexts_corpus; - contexts_corpus.read_contexts(vm["data"].as<string>(), backoff_gen, /*vm.count("filter-singleton-contexts")*/ false); - model.set_backoff(contexts_corpus.backoff_index()); - - if (backoff_gen) - delete backoff_gen; - - // train the sampler - model.sample_corpus(contexts_corpus, vm["samples"].as<int>(), - vm["freq-cutoff-start"].as<int>(), - vm["freq-cutoff-end"].as<int>(), - vm["freq-cutoff-interval"].as<int>(), - vm["max-contexts-per-document"].as<int>(), - vm["temp-start"].as<double>(), vm["temp-end"].as<double>()); - - if (vm.count("document-topics-out")) { - ogzstream documents_out(vm["document-topics-out"].as<string>().c_str()); - - int document_id=0; - map<int,int> all_terms; - for (Corpus::const_iterator corpusIt=contexts_corpus.begin(); - corpusIt != contexts_corpus.end(); ++corpusIt, ++document_id) { - vector<int> unique_terms; - for (Document::const_iterator docIt=corpusIt->begin(); - docIt != corpusIt->end(); ++docIt) { - if (unique_terms.empty() || *docIt != unique_terms.back()) - unique_terms.push_back(*docIt); - // increment this terms frequency - pair<map<int,int>::iterator,bool> insert_result = all_terms.insert(make_pair(*docIt,1)); - if (!insert_result.second) - all_terms[*docIt] = all_terms[*docIt] + 1; - //insert_result.first++; - } - documents_out << contexts_corpus.key(document_id) << '\t'; - documents_out << model.max(document_id).first << " " << corpusIt->size() << " ||| "; - for (std::vector<int>::const_iterator termIt=unique_terms.begin(); - termIt != unique_terms.end(); ++termIt) { - if (termIt != unique_terms.begin()) - documents_out << " ||| "; - vector<std::string> strings = contexts_corpus.context2string(*termIt); - copy(strings.begin(), strings.end(),ostream_iterator<std::string>(documents_out, " ")); - std::pair<int,PYPTopics::F> maxinfo = model.max(document_id, *termIt); - documents_out << "||| C=" << maxinfo.first << " P=" << maxinfo.second; - - } - documents_out <<endl; - } - documents_out.close(); - - if (vm.count("default-topics-out")) { - ofstream default_topics(vm["default-topics-out"].as<string>().c_str()); - default_topics << model.max_topic() <<endl; - for (std::map<int,int>::const_iterator termIt=all_terms.begin(); termIt != all_terms.end(); ++termIt) { - vector<std::string> strings = contexts_corpus.context2string(termIt->first); - default_topics << model.max(-1, termIt->first).first << " ||| " << termIt->second << " ||| "; - copy(strings.begin(), strings.end(),ostream_iterator<std::string>(default_topics, " ")); - default_topics <<endl; - } - } - } - - if (vm.count("topic-words-out")) { - ogzstream topics_out(vm["topic-words-out"].as<string>().c_str()); - model.print_topic_terms(topics_out); - topics_out.close(); - } - - cout <<endl; - - return 0; -} diff --git a/gi/pyp-topics/src/train.cc b/gi/pyp-topics/src/train.cc deleted file mode 100644 index db7ca46e..00000000 --- a/gi/pyp-topics/src/train.cc +++ /dev/null @@ -1,135 +0,0 @@ -// STL -#include <iostream> -#include <fstream> - -// Boost -#include <boost/program_options/parsers.hpp> -#include <boost/program_options/variables_map.hpp> -#include <boost/scoped_ptr.hpp> - -// Local -#include "pyp-topics.hh" -#include "corpus.hh" -#include "contexts_corpus.hh" -#include "gzstream.hh" - -static const char *REVISION = "$Rev$"; - -// Namespaces -using namespace boost; -using namespace boost::program_options; -using namespace std; - -int main(int argc, char **argv) -{ - std::cout << "Pitman Yor topic models: Copyright 2010 Phil Blunsom\n"; - std::cout << REVISION << '\n' << std::endl; - - //////////////////////////////////////////////////////////////////////////////////////////// - // Command line processing - variables_map vm; - - // Command line processing - options_description cmdline_specific("Command line specific options"); - cmdline_specific.add_options() - ("help,h", "print help message") - ("config,c", value<string>(), "config file specifying additional command line options") - ; - options_description generic("Allowed options"); - generic.add_options() - ("documents,d", value<string>(), "file containing the documents") - ("topics,t", value<int>()->default_value(50), "number of topics") - ("document-topics-out,o", value<string>(), "file to write the document topics to") - ("topic-words-out,w", value<string>(), "file to write the topic word distribution to") - ("samples,s", value<int>()->default_value(10), "number of sampling passes through the data") - ("test-corpus", value<string>(), "file containing the test data") - ("backoff-paths", value<string>(), "file containing the term backoff paths") - ; - options_description config_options, cmdline_options; - config_options.add(generic); - cmdline_options.add(generic).add(cmdline_specific); - - store(parse_command_line(argc, argv, cmdline_options), vm); - if (vm.count("config") > 0) { - ifstream config(vm["config"].as<string>().c_str()); - store(parse_config_file(config, cmdline_options), vm); - } - notify(vm); - //////////////////////////////////////////////////////////////////////////////////////////// - - if (vm.count("documents") == 0) { - cerr << "Please specify a file containing the documents." << endl; - cout << cmdline_options << "\n"; - return 1; - } - - if (vm.count("help")) { - cout << cmdline_options << "\n"; - return 1; - } - - // seed the random number generator: 0 = automatic, specify value otherwise - unsigned long seed = 0; - PYPTopics model(vm["topics"].as<int>(), false, seed); - - // read the data - Corpus corpus; - corpus.read(vm["documents"].as<string>()); - - // read the backoff dictionary - if (vm.count("backoff-paths")) - model.set_backoff(vm["backoff-paths"].as<string>()); - - // train the sampler - model.sample_corpus(corpus, vm["samples"].as<int>()); - - if (vm.count("document-topics-out")) { - ogzstream documents_out(vm["document-topics-out"].as<string>().c_str()); - //model.print_document_topics(documents_out); - - int document_id=0; - for (Corpus::const_iterator corpusIt=corpus.begin(); - corpusIt != corpus.end(); ++corpusIt, ++document_id) { - std::vector<int> unique_terms; - for (Document::const_iterator docIt=corpusIt->begin(); - docIt != corpusIt->end(); ++docIt) { - if (unique_terms.empty() || *docIt != unique_terms.back()) - unique_terms.push_back(*docIt); - } - documents_out << unique_terms.size(); - for (std::vector<int>::const_iterator termIt=unique_terms.begin(); - termIt != unique_terms.end(); ++termIt) - documents_out << " " << *termIt << ":" << model.max(document_id, *termIt).first; - documents_out << std::endl; - } - documents_out.close(); - } - - if (vm.count("topic-words-out")) { - ogzstream topics_out(vm["topic-words-out"].as<string>().c_str()); - model.print_topic_terms(topics_out); - topics_out.close(); - } - - if (vm.count("test-corpus")) { - TestCorpus test_corpus; - test_corpus.read(vm["test-corpus"].as<string>()); - ogzstream topics_out((vm["test-corpus"].as<string>() + ".topics.gz").c_str()); - - for (TestCorpus::const_iterator corpusIt=test_corpus.begin(); - corpusIt != test_corpus.end(); ++corpusIt) { - int index=0; - for (DocumentTerms::const_iterator instanceIt=corpusIt->begin(); - instanceIt != corpusIt->end(); ++instanceIt, ++index) { - int topic = model.max(instanceIt->doc, instanceIt->term).first; - if (index != 0) topics_out << " "; - topics_out << topic; - } - topics_out << std::endl; - } - topics_out.close(); - } - std::cout << std::endl; - - return 0; -} diff --git a/gi/pyp-topics/src/utility.h b/gi/pyp-topics/src/utility.h deleted file mode 100644 index 405a5b0a..00000000 --- a/gi/pyp-topics/src/utility.h +++ /dev/null @@ -1,962 +0,0 @@ -// utility.h -// -// (c) Mark Johnson, 24th January 2005 -// -// modified 6th May 2002 to ensure write/read consistency, fixed 18th July 2002 -// modified 14th July 2002 to include insert() (generic inserter) -// modified 26th September 2003 to use mapped_type instead of data_type -// 25th August 2004 added istream >> const char* -// 24th January 2005 added insert_newkey() -// -// Defines: -// loop macros foreach, cforeach -// dfind (default find function) -// afind (find function that asserts key exists) -// insert_newkey (inserts a new key into a map) -// insert (generic inserter into standard data structures) -// disjoint (set operation) -// first_lessthan and second_lessthan (compares elements of pairs) -// -// Simplified interfaces to STL routines: -// -// includes (simplified interface) -// set_intersection (simplified interface) -// inserter (simplified interface) -// max_element (simplified interface) -// min_element (simplified interface) -// hash functions for pairs, vectors, lists, slists and maps -// input and output for pairs and vectors -// resource_usage (interface improved) - - -#ifndef UTILITY_H -#define UTILITY_H - -#include <algorithm> -// #include <boost/smart_ptr.hpp> // Comment out this line if boost is not used -#include <cassert> -#include <cmath> -#include <cctype> -#include <cstdio> -#include <unordered_map> -#include <unordered_set> -#include <ext/slist> -#include <iostream> -#include <iterator> -#include <list> -#include <map> -#include <set> -#include <string> -#include <utility> -#include <vector> -#include <memory> - -#if (__GNUC__ > 3) || (__GNUC__ >= 3 && __GNUC_MINOR__ >= 1) -#define EXT_NAMESPACE __gnu_cxx -#else -#define EXT_NAMESPACE std -#endif - -namespace ext = EXT_NAMESPACE; - -inline float power(float x, float y) { return powf(x, y); } -inline double power(double x, double y) { return pow(x, y); } -inline long double power(long double x, long double y) { return powl(x, y); } - -typedef unsigned U; -typedef long double F; // slower than double, but underflows less - -/////////////////////////////////////////////////////////////////////////// -// // -// Looping constructs // -// // -/////////////////////////////////////////////////////////////////////////// - -// foreach is a simple loop construct -// -// STORE should be an STL container -// TYPE is the typename of STORE -// VAR will be defined as a local variable of type TYPE::iterator -// -#define foreach(TYPE, VAR, STORE) \ - for (TYPE::iterator VAR = (STORE).begin(); VAR != (STORE).end(); ++VAR) - -// cforeach is just like foreach, except that VAR is a const_iterator -// -// STORE should be an STL container -// TYPE is the typename of STORE -// VAR will be defined as a local variable of type TYPE::const_iterator -// -#define cforeach(TYPE, VAR, STORE) \ - for (TYPE::const_iterator VAR = (STORE).begin(); VAR != (STORE).end(); ++VAR) - - -/////////////////////////////////////////////////////////////////////////// -// // -// Map searching // -// // -// dfind(map, key) returns the key's value in map, or map's default // -// value if no such key exists (the default value is not inserted) // -// // -// afind(map, key) returns a reference to the key's value in map, and // -// asserts that this value exists // -// // -/////////////////////////////////////////////////////////////////////////// - -// dfind(Map, Key) returns the value Map associates with Key, or the -// Map's default value if no such Key exists -// -template <class Map, class Key> -inline typename Map::mapped_type dfind(Map& m, const Key& k) -{ - typename Map::iterator i = m.find(k); - if (i == m.end()) - return typename Map::mapped_type(); - else - return i->second; -} - -template <class Map, class Key> -inline const typename Map::mapped_type dfind(const Map& m, const Key& k) -{ - typename Map::const_iterator i = m.find(k); - if (i == m.end()) - return typename Map::mapped_type(); - else - return i->second; -} - - -// afind(map, key) returns a reference to the value associated -// with key in map. It uses assert to check that the key's value -// is defined. -// -template <class Map, class Key> -inline typename Map::mapped_type& afind(Map& m, const Key& k) -{ - typename Map::iterator i = m.find(k); - assert(i != m.end()); - return i->second; -} - -template <class Map, class Key> -inline const typename Map::mapped_type& afind(const Map& m, const Key& k) -{ - typename Map::const_iterator i = m.find(k); - assert(i != m.end()); - return i->second; -} - -//! insert_newkey(map, key, value) checks that map does not contain -//! key, and binds key to value. -// -template <class Map, class Key, class Value> -inline typename Map::value_type& -insert_newkey(Map& m, const Key& k,const Value& v) -{ - std::pair<typename Map::iterator, bool> itb - = m.insert(Map::value_type(k, v)); - assert(itb.second); - return *(itb.first); -} // insert_newkey() - - -/////////////////////////////////////////////////////////////////////////// -// // -// Insert operations // -// // -/////////////////////////////////////////////////////////////////////////// - - -template <typename T> -void insert(std::list<T>& xs, const T& x) { - xs.push_back(x); -} - -template <typename T> -void insert(std::set<T>& xs, const T& x) { - xs.insert(x); -} - -template <typename T> -void insert(std::vector<T>& xs, const T& x) { - xs.push_back(x); -} - - -/////////////////////////////////////////////////////////////////////////// -// // -// Additional versions of standard algorithms // -// // -/////////////////////////////////////////////////////////////////////////// - -template <typename Set1, typename Set2> -inline bool includes(const Set1& set1, const Set2& set2) -{ - return std::includes(set1.begin(), set1.end(), set2.begin(), set2.end()); -} - -template <typename Set1, typename Set2, typename Compare> -inline bool includes(const Set1& set1, const Set2& set2, Compare comp) -{ - return std::includes(set1.begin(), set1.end(), set2.begin(), set2.end(), comp); -} - - -template <typename InputIter1, typename InputIter2> -bool disjoint(InputIter1 first1, InputIter1 last1, - InputIter2 first2, InputIter2 last2) -{ - while (first1 != last1 && first2 != last2) - if (*first1 < *first2) - ++first1; - else if (*first2 < *first1) - ++first2; - else // *first1 == *first2 - return false; - return true; -} - -template <typename InputIter1, typename InputIter2, typename Compare> -bool disjoint(InputIter1 first1, InputIter1 last1, - InputIter2 first2, InputIter2 last2, Compare comp) -{ - while (first1 != last1 && first2 != last2) - if (comp(*first1, *first2)) - ++first1; - else if (comp(*first2, *first1)) - ++first2; - else // *first1 == *first2 - return false; - return true; -} - -template <typename Set1, typename Set2> -inline bool disjoint(const Set1& set1, const Set2& set2) -{ - return disjoint(set1.begin(), set1.end(), set2.begin(), set2.end()); -} - -template <typename Set1, typename Set2, typename Compare> -inline bool disjoint(const Set1& set1, const Set2& set2, Compare comp) -{ - return disjoint(set1.begin(), set1.end(), set2.begin(), set2.end(), comp); -} - - -template <typename Set1, typename Set2, typename OutputIterator> -inline OutputIterator set_intersection(const Set1& set1, const Set2& set2, - OutputIterator result) -{ - return set_intersection(set1.begin(), set1.end(), set2.begin(), set2.end(), result); -} - -template <typename Set1, typename Set2, typename OutputIterator, typename Compare> -inline OutputIterator set_intersection(const Set1& set1, const Set2& set2, - OutputIterator result, Compare comp) -{ - return set_intersection(set1.begin(), set1.end(), set2.begin(), set2.end(), result, comp); -} - - -template <typename Container> -inline std::insert_iterator<Container> inserter(Container& container) -{ - return std::inserter(container, container.begin()); -} - -// max_element -// -template <class Es> inline typename Es::iterator max_element(Es& es) -{ - return std::max_element(es.begin(), es.end()); -} - -template <class Es> inline typename Es::const_iterator max_element(const Es& es) -{ - return std::max_element(es.begin(), es.end()); -} - -template <class Es, class BinaryPredicate> -inline typename Es::iterator max_element(Es& es, BinaryPredicate comp) -{ - return std::max_element(es.begin(), es.end(), comp); -} - -template <class Es, class BinaryPredicate> -inline typename Es::const_iterator max_element(const Es& es, BinaryPredicate comp) -{ - return std::max_element(es.begin(), es.end(), comp); -} - -// min_element -// -template <class Es> inline typename Es::iterator min_element(Es& es) -{ - return std::min_element(es.begin(), es.end()); -} - -template <class Es> inline typename Es::const_iterator min_element(const Es& es) -{ - return std::min_element(es.begin(), es.end()); -} - -template <class Es, class BinaryPredicate> -inline typename Es::iterator min_element(Es& es, BinaryPredicate comp) -{ - return std::min_element(es.begin(), es.end(), comp); -} - -template <class Es, class BinaryPredicate> -inline typename Es::const_iterator min_element(const Es& es, BinaryPredicate comp) -{ - return std::min_element(es.begin(), es.end(), comp); -} - -// first_lessthan and second_lessthan -// -struct first_lessthan { - template <typename T1, typename T2> - bool operator() (const T1& e1, const T2& e2) { - return e1.first < e2.first; - } -}; - -struct second_lessthan { - template <typename T1, typename T2> - bool operator() (const T1& e1, const T2& e2) { - return e1.second < e2.second; - } -}; - -// first_greaterthan and second_greaterthan -// -struct first_greaterthan { - template <typename T1, typename T2> - bool operator() (const T1& e1, const T2& e2) { - return e1.first > e2.first; - } -}; - -struct second_greaterthan { - template <typename T1, typename T2> - bool operator() (const T1& e1, const T2& e2) { - return e1.second > e2.second; - } -}; - - -/////////////////////////////////////////////////////////////////////////// -// // -// hash<> specializations // -// // -// These must be in namespace std. They permit the corresponding STL // -// container to be used as a key in an STL hash table. // -// // -/////////////////////////////////////////////////////////////////////////// - -//namespace EXT_NAMESPACE { -namespace std { - /* - // hash function for bool - // - template <> struct hash<bool> - { - size_t operator() (bool b) const - { - return b; - } // operator() - }; // hash<bool>{} - - // hash function for double - // - template <> struct hash<double> - { - size_t operator() (double d) const - { - int exponent; - double fraction = frexp(d, &exponent); - return size_t(exponent) ^ size_t(1000000.0*(fabs(fraction-0.5))); - } // operator() - }; // hash<double>{} - - // hash function for strings - // - template <> struct hash<std::string> - { - size_t operator()(const std::string& s) const - { - typedef std::string::const_iterator CI; - - unsigned long h = 0; - unsigned long g; - CI p = s.begin(); - CI end = s.end(); - - while (p!=end) { - h = (h << 4) + (*p++); - if ((g = h&0xf0000000)) { - h = h ^ (g >> 24); - h = h ^ g; - }} - return size_t(h); - } // operator() - }; // hash<string>{} - -*/ - // hash function for arbitrary pairs - // - template<class T1, class T2> struct hash<std::pair<T1,T2> > { - size_t operator()(const std::pair<T1,T2>& p) const - { - size_t h1 = hash<T1>()(p.first); - size_t h2 = hash<T2>()(p.second); - return h1 ^ (h1 >> 1) ^ h2 ^ (h2 << 1); - } - }; - - - // hash function for vectors - // - template<class T> struct hash<std::vector<T> > - { // This is the fn hashpjw of Aho, Sethi and Ullman, p 436. - size_t operator()(const std::vector<T>& s) const - { - typedef typename std::vector<T>::const_iterator CI; - - unsigned long h = 0; - unsigned long g; - CI p = s.begin(); - CI end = s.end(); - - while (p!=end) { - h = (h << 5) + hash<T>()(*p++); - if ((g = h&0xff000000)) { - h = h ^ (g >> 23); - h = h ^ g; - }} - return size_t(h); - } - }; - - // hash function for slists - // - template<class T> struct hash<ext::slist<T> > - { // This is the fn hashpjw of Aho, Sethi and Ullman, p 436. - size_t operator()(const ext::slist<T>& s) const - { - typedef typename ext::slist<T>::const_iterator CI; - - unsigned long h = 0; - unsigned long g; - CI p = s.begin(); - CI end = s.end(); - - while (p!=end) { - h = (h << 7) + hash<T>()(*p++); - if ((g = h&0xff000000)) { - h = h ^ (g >> 23); - h = h ^ g; - }} - return size_t(h); - } - }; - - // hash function for maps - // - template<typename T1, typename T2> struct hash<std::map<T1,T2> > - { - size_t operator()(const std::map<T1,T2>& m) const - { - typedef typename std::map<T1,T2> M; - typedef typename M::const_iterator CI; - - unsigned long h = 0; - unsigned long g; - CI p = m.begin(); - CI end = m.end(); - - while (p != end) { - h = (h << 11) + hash<typename M::value_type>()(*p++); - if ((g = h&0xff000000)) { - h = h ^ (g >> 23); - h = h ^ g; - }} - return size_t(h); - } - }; - -} // namespace EXT_NAMESPACE - - - -/////////////////////////////////////////////////////////////////////////// -// // -// Write/Read code // -// // -// These routines should possess write/read invariance IF their elements // -// also have write-read invariance. Whitespace, '(' and ')' are used as // -// delimiters. // -// // -/////////////////////////////////////////////////////////////////////////// - - -// Define istream >> const char* so that it consumes the characters from the -// istream. Just as in scanf, a space consumes an arbitrary amount of whitespace. -// -inline std::istream& operator>> (std::istream& is, const char* cp) -{ - if (*cp == '\0') - return is; - else if (*cp == ' ') { - char c; - if (is.get(c)) { - if (isspace(c)) - return is >> cp; - else { - is.unget(); - return is >> (cp+1); - } - } - else { - is.clear(is.rdstate() & ~std::ios::failbit); // clear failbit - return is >> (cp+1); - } - } - else { - char c; - if (is.get(c)) { - if (c == *cp) - return is >> (cp+1); - else { - is.unget(); - is.setstate(std::ios::failbit); - } - } - return is; - } -} - - -// Write out an auto_ptr object just as you would write out the pointer object -// -template <typename T> -inline std::ostream& operator<<(std::ostream& os, const std::auto_ptr<T>& sp) -{ - return os << sp.get(); -} - - -// Pairs -// -template <class T1, class T2> -std::ostream& operator<< (std::ostream& os, const std::pair<T1,T2>& p) -{ - return os << '(' << p.first << ' ' << p.second << ')'; -} - -template <class T1, class T2> -std::istream& operator>> (std::istream& is, std::pair<T1,T2>& p) -{ - char c; - if (is >> c) { - if (c == '(') { - if (is >> p.first >> p.second >> c && c == ')') - return is; - else - is.setstate(std::ios::badbit); - } - else - is.putback(c); - } - is.setstate(std::ios::failbit); - return is; -} - -// Lists -// -template <class T> -std::ostream& operator<< (std::ostream& os, const std::list<T>& xs) -{ - os << '('; - for (typename std::list<T>::const_iterator xi = xs.begin(); xi != xs.end(); ++xi) { - if (xi != xs.begin()) - os << ' '; - os << *xi; - } - return os << ')'; -} - -template <class T> -std::istream& operator>> (std::istream& is, std::list<T>& xs) -{ - char c; // This code avoids unnecessary copy - if (is >> c) { // read the initial '(' - if (c == '(') { - xs.clear(); // clear the list - do { - xs.push_back(T()); // create a new elt in list - is >> xs.back(); // read element - } - while (is.good()); // read as long as possible - xs.pop_back(); // last read failed; pop last elt - is.clear(is.rdstate() & ~std::ios::failbit); // clear failbit - if (is >> c && c == ')') // read terminating ')' - return is; // successful return - else - is.setstate(std::ios::badbit); // something went wrong, set badbit - } - else // c is not '(' - is.putback(c); // put c back into input - } - is.setstate(std::ios::failbit); // read failed, set failbit - return is; -} - -// Vectors -// -template <class T> -std::ostream& operator<< (std::ostream& os, const std::vector<T>& xs) -{ - os << '('; - for (typename std::vector<T>::const_iterator xi = xs.begin(); xi != xs.end(); ++xi) { - if (xi != xs.begin()) - os << ' '; - os << *xi; - } - return os << ')'; -} - -template <class T> -std::istream& operator>> (std::istream& is, std::vector<T>& xs) -{ - char c; // This code avoids unnecessary copy - if (is >> c) { // read the initial '(' - if (c == '(') { - xs.clear(); // clear the list - do { - xs.push_back(T()); // create a new elt in list - is >> xs.back(); // read element - } - while (is.good()); // read as long as possible - xs.pop_back(); // last read failed; pop last elt - is.clear(is.rdstate() & ~std::ios::failbit); // clear failbit - if (is >> c && c == ')') // read terminating ')' - return is; // successful return - else - is.setstate(std::ios::badbit); // something went wrong, set badbit - } - else // c is not '(' - is.putback(c); // put c back into input - } - is.setstate(std::ios::failbit); // read failed, set failbit - return is; -} - -// Slists -// -template <class T> -std::ostream& operator<< (std::ostream& os, const ext::slist<T>& xs) -{ - os << '('; - for (typename ext::slist<T>::const_iterator xi = xs.begin(); xi != xs.end(); ++xi) { - if (xi != xs.begin()) - os << ' '; - os << *xi; - } - return os << ')'; -} - -template <class T> -std::istream& operator>> (std::istream& is, ext::slist<T>& xs) -{ - char c; - if (is >> c) { - if (c == '(') { - xs.clear(); - T e; - if (is >> e) { - xs.push_front(e); - typename ext::slist<T>::iterator xi = xs.begin(); - while (is >> e) - xi = xs.insert_after(xi, e); - is.clear(is.rdstate() & ~std::ios::failbit); - if (is >> c && c == ')') - return is; - else - is.setstate(std::ios::badbit); - } - else { // empty list - is.clear(is.rdstate() & ~std::ios::failbit); - if (is >> c && c == ')') - return is; - else // didn't see closing ')' - is.setstate(std::ios::badbit); - } - } - else // didn't read '(' - is.putback(c); - } - is.setstate(std::ios::failbit); - return is; -} - -// Sets -// -template <class T> -std::ostream& operator<< (std::ostream& os, const std::set<T>& s) -{ - os << '('; - for (typename std::set<T>::const_iterator i = s.begin(); i != s.end(); ++i) { - if (i != s.begin()) - os << ' '; - os << *i; - } - return os << ')'; -} - -template <class T> -std::istream& operator>> (std::istream& is, std::set<T>& s) -{ - char c; - if (is >> c) { - if (c == '(') { - s.clear(); - T e; - while (is >> e) - s.insert(e); - is.clear(is.rdstate() & ~std::ios::failbit); - if (is >> c && c == ')') - return is; - else - is.setstate(std::ios::badbit); - } - else - is.putback(c); - } - is.setstate(std::ios::failbit); - return is; -} - -// Hash_sets -// -template <class T> -std::ostream& operator<< (std::ostream& os, const std::unordered_set<T>& s) -{ - os << '('; - for (typename std::unordered_set<T>::const_iterator i = s.begin(); i != s.end(); ++i) { - if (i != s.begin()) - os << ' '; - os << *i; - } - return os << ')'; -} - -template <class T> -std::istream& operator>> (std::istream& is, std::unordered_set<T>& s) -{ - char c; - if (is >> c) { - if (c == '(') { - s.clear(); - T e; - while (is >> e) - s.insert(e); - is.clear(is.rdstate() & ~std::ios::failbit); - if (is >> c && c == ')') - return is; - else - is.setstate(std::ios::badbit); - } - else - is.putback(c); - } - is.setstate(std::ios::failbit); - return is; -} - - -// Maps -// -template <class Key, class Value> -std::ostream& operator<< (std::ostream& os, const std::map<Key,Value>& m) -{ - typedef std::map<Key,Value> M; - os << '('; - for (typename M::const_iterator it = m.begin(); it != m.end(); ++it) { - if (it != m.begin()) - os << ' '; - os << *it; - } - return os << ")"; -} - -template <class Key, class Value> -std::istream& operator>> (std::istream& is, std::map<Key,Value>& m) -{ - char c; - if (is >> c) { - if (c == '(') { - m.clear(); - std::pair<Key,Value> e; - while (is >> e) - m.insert(e); - is.clear(is.rdstate() & ~std::ios::failbit); - if (is >> c && c == ')') - return is; - else - is.setstate(std::ios::badbit); - } - else - is.putback(c); - } - is.setstate(std::ios::failbit); - return is; -} - -// Hash_maps -// -template <class Key, class Value> -std::ostream& operator<< (std::ostream& os, const std::unordered_map<Key,Value>& m) -{ - typedef std::unordered_map<Key,Value> M; - os << '('; - for (typename M::const_iterator it = m.begin(); it != m.end(); ++it) { - if (it != m.begin()) - os << ' '; - os << *it; - } - return os << ")"; -} - -template <class Key, class Value> -std::istream& operator>> (std::istream& is, std::unordered_map<Key,Value>& m) -{ - char c; - if (is >> c) { - if (c == '(') { - m.clear(); - std::pair<Key,Value> e; - while (is >> e) - m.insert(e); - is.clear(is.rdstate() & ~std::ios::failbit); - if (is >> c && c == ')') - return is; - else - is.setstate(std::ios::badbit); - } - else - is.putback(c); - } - is.setstate(std::ios::failbit); - return is; -} - - -/////////////////////////////////////////////////////////////////////////// -// // -// Boost library additions // -// // -/////////////////////////////////////////////////////////////////////////// - -#ifdef BOOST_SHARED_PTR_HPP_INCLUDED - -// enhancements to boost::shared_ptr so it can be used with hash -// -namespace std { - template <typename T> struct equal_to<boost::shared_ptr<T> > - : public binary_function<boost::shared_ptr<T>, boost::shared_ptr<T>, bool> { - bool operator() (const boost::shared_ptr<T>& p1, const boost::shared_ptr<T>& p2) const { - return equal_to<T*>()(p1.get(), p2.get()); - } - }; -} // namespace std - -//namespace EXT_NAMESPACE { -namespace std { - template <typename T> struct hash<boost::shared_ptr<T> > { - size_t operator() (const boost::shared_ptr<T>& a) const { - return hash<T*>()(a.get()); - } - }; -} // namespace ext - -template <typename T> -inline std::ostream& operator<< (std::ostream& os, const boost::shared_ptr<T>& sp) -{ - return os << sp.get(); -} - -#endif // BOOST_SHARED_PTR_HPP_INCLUDED - -struct resource_usage { }; - -#ifndef __i386 -inline std::ostream& operator<< (std::ostream& os, resource_usage r) -{ - return os; -} -#else // Assume we are on a 586 linux -inline std::ostream& operator<< (std::ostream& os, resource_usage r) -{ - FILE* fp = fopen("/proc/self/stat", "r"); - assert(fp); - int utime; - int stime; - unsigned int vsize; - unsigned int rss; - int result = - fscanf(fp, "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %d %d %*d %*d %*d %*d" - "%*u %*u %*d %u %u", &utime, &stime, &vsize, &rss); - assert(result == 4); - fclose(fp); - // s << "utime = " << utime << ", stime = " << stime << ", vsize = " << vsize << ", rss = " << rss - ; - // return s << "utime = " << utime << ", vsize = " << vsize; - return os << "utime " << float(utime)/1.0e2 << "s, vsize " - << float(vsize)/1048576.0 << " Mb."; -} -#endif - -//! A default_value_type{} object is used to read an object from a stream, -//! assigning a default value if the read fails. Users should not need to -//! construct such objects, but should use default_value() instead. -// -template <typename object_type, typename default_type> -struct default_value_type { - object_type& object; - const default_type defaultvalue; - default_value_type(object_type& object, const default_type defaultvalue) - : object(object), defaultvalue(defaultvalue) { } -}; - -//! default_value() is used to read an object from a stream, assigning a -//! default value if the read fails. It returns a default_value_type{} -//! object, which does the actual reading. -// -template <typename object_type, typename default_type> -default_value_type<object_type,default_type> -default_value(object_type& object, const default_type defaultvalue=default_type()) { - return default_value_type<object_type,default_type>(object, defaultvalue); -} - -//! This version of operator>>() reads default_value_type{} from an input stream. -// -template <typename object_type, typename default_type> -std::istream& operator>> (std::istream& is, - default_value_type<object_type, default_type> dv) { - if (is) { - if (is >> dv.object) - ; - else { - is.clear(is.rdstate() & ~std::ios::failbit); // clear failbit - dv.object = dv.defaultvalue; - } - } - return is; -} - -// inline F random1() { return rand()/(RAND_MAX+1.0); } -inline F random1() { return mt_genrand_res53(); } - -#endif // UTILITY_H diff --git a/gi/pyp-topics/src/workers.hh b/gi/pyp-topics/src/workers.hh deleted file mode 100644 index 95b18947..00000000 --- a/gi/pyp-topics/src/workers.hh +++ /dev/null @@ -1,275 +0,0 @@ -/** - Basic thread-pool tools using Boost.Thread. - (Jan Botha, 7/2010) - - --Simple usage-- - Use SimpleWorker. - Example, call a function that returns an int in a new thread: - typedef boost::function<int()> JobType; - JobType job = boost::bind(funcname); - //or boost::bind(&class::funcname, this) for a member function - SimpleWorker<JobType, int> worker(job); - int result = worker.getResult(); //blocks until result is ready - - --Extended usage-- - Use WorkerPool, which uses Queuemt (a synchronized queue) and Worker. - Example: - (same context and typedef - WorkerPool<JobType, int> pool(num_threads); - JobType job = ... - pool.addJob(job); - ... - pool.get_result(); //blocks until all workers are done, returns the some of their results. - - Jobs added to a WorkerPool need to be the same type. A WorkerPool instance should not be reused (e.g. adding jobs) after calling get_result(). -*/ - -#ifndef WORKERS_HH -#define WORKERS_HH - -#include <iostream> -#include <boost/bind.hpp> -#include <boost/function.hpp> -#include <queue> -#include <boost/ptr_container/ptr_vector.hpp> -#include <boost/thread/thread.hpp> -#include <boost/thread/mutex.hpp> -#include <boost/thread/shared_mutex.hpp> -#include <boost/thread/future.hpp> -#include <boost/thread/condition.hpp> - -#include <boost/date_time/posix_time/posix_time_types.hpp> -#include "timing.h" - -/** Implements a synchronized queue*/ -template<typename J> -class Queuemt -{ - -public: - boost::condition_variable_any cond; - const bool& running; - - Queuemt() { } - Queuemt(const bool& running) : running(running), maxsize(0), qsize(0) - { - } - - ~Queuemt() { - } - - J pop() - { - J job; - { - boost::unique_lock<boost::shared_mutex> qlock(q_mutex); - while (running && qsize == 0) - cond.wait(qlock); - - if (qsize > 0) - { - job = q.front(); - q.pop(); - --qsize; - } - } - if (job) - cond.notify_one(); - return job; - - } - - void push(J job) - { - { - boost::unique_lock<boost::shared_mutex> lock(q_mutex); - q.push(job); - ++qsize; - } - if (qsize > maxsize) - maxsize = qsize; - - cond.notify_one(); - } - - int getMaxsize() - { - return maxsize; - } - - int size() - { - return qsize; - } - -private: - boost::shared_mutex q_mutex; - std::queue<J> q; - int maxsize; - volatile int qsize; -}; - - -template<typename J, typename R> -class Worker -{ -typedef boost::packaged_task<R> PackagedTask; -public: - Worker(Queuemt<J>& queue, int id, int num_workers) : - q(queue), tasktime(0.0), id(id), num_workers(num_workers) - { - PackagedTask task(boost::bind(&Worker<J, R>::run, this)); - future = task.get_future(); - boost::thread t(boost::move(task)); - } - - R run() //this is called upon thread creation - { - R wresult = 0; - while (isRunning()) - { - J job = q.pop(); - - if (job) - { - timer.Reset(); - wresult += job(); - tasktime += timer.Elapsed(); - } - } - return wresult; - } - - R getResult() - { - if (!future.is_ready()) - future.wait(); - assert(future.is_ready()); - return future.get(); - } - - double getTaskTime() - { - return tasktime; - } - -private: - - Queuemt<J>& q; - - boost::unique_future<R> future; - - bool isRunning() - { - return q.running || q.size() > 0; - } - - Timer timer; - double tasktime; - int id; - int num_workers; -}; - -template<typename J, typename R> -class WorkerPool -{ -typedef boost::packaged_task<R> PackagedTask; -typedef Worker<J,R> WJR; -typedef boost::ptr_vector<WJR> WorkerVector; -public: - - WorkerPool(int num_workers) - { - q.reset(new Queuemt<J>(running)); - running = true; - for (int i = 0; i < num_workers; ++i) - workers.push_back( new Worker<J, R>(*q, i, num_workers) ); - } - - ~WorkerPool() - { - } - - R get_result() - { - running = false; - q->cond.notify_all(); - R tmp = 0; - double tasktime = 0.0; - for (typename WorkerVector::iterator it = workers.begin(); it != workers.end(); it++) - { - R res = it->getResult(); - tmp += res; - //std::cerr << "tasktime: " << it->getTaskTime() << std::endl; - tasktime += it->getTaskTime(); - } -// std::cerr << " maxQ = " << q->getMaxsize() << std::endl; - return tmp; - } - - void addJob(J job) - { - q->push(job); - } - -private: - - WorkerVector workers; - - boost::shared_ptr<Queuemt<J> > q; - - bool running; -}; - -/////////////////// -template <typename J, typename R> -class SimpleWorker -{ -typedef boost::packaged_task<R> PackagedTask; -public: - SimpleWorker(J& job) : job(job), tasktime(0.0) - { - PackagedTask task(boost::bind(&SimpleWorker<J, R>::run, this)); - future = task.get_future(); - boost::thread t(boost::move(task)); - } - - R run() //this is called upon thread creation - { - R wresult = 0; - - assert(job); - timer.Reset(); - wresult = job(); - tasktime = timer.Elapsed(); - std::cerr << tasktime << " s" << std::endl; - return wresult; - } - - R getResult() - { - if (!future.is_ready()) - future.wait(); - assert(future.is_ready()); - return future.get(); - } - - double getTaskTime() - { - return tasktime; - } - -private: - - J job; - - boost::unique_future<R> future; - - Timer timer; - double tasktime; - -}; - - - -#endif diff --git a/gi/scripts/buck2utf8.pl b/gi/scripts/buck2utf8.pl deleted file mode 100755 index 1acfae8d..00000000 --- a/gi/scripts/buck2utf8.pl +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/perl -w -use strict; -use utf8; -binmode(STDOUT, ":utf8"); -while(<>) { - chomp; - my @words = split /\s+/; - for my $w (@words) { - $_ = $w; - if ($w =~ /^__NTK__/o) { - s/__NTK__//go; - next if /^$/; - print STDOUT "$_ "; - next; - } -s/tR/\x{0679}/g; # retroflex t -s/dR/\x{0688}/g; # retroflex d -s/rR/\x{0691}/g; # retroflex r -s/p/\x{067E}/g; # peh -s/c/\x{0686}/g; # tcheh -s/g/\x{06AF}/g; # geh (G=ghain) -s/@/\x{06BE}/g; # heh doachashmee -s/h'/\x{06c2}/g; # heh goal + hamza -s/h/\x{06c1}/g; # heh goal -s/J/\x{0698}/g; # zheh (rare, usually persian loan words) -s/k/\x{06A9}/g; # k -s/Y'/\x{06d3}/g; # yeh barree + hamza above (ligature) -s/y/\x{06cc}/g; # same as ya' in arabic -s/Y/\x{06d2}/g; # yeh barree -s/N/\x{06BA}/g; # Ghunna - - s/\'/\x{0621}/g; - s/\|/\x{0622}/g; - s/\>/\x{0623}/g; - s/\&/\x{0624}/g; - s/\</\x{0625}/g; - s/\}/\x{0626}/g; - s/A/\x{0627}/g; - s/b/\x{0628}/g; - s/t/\x{062A}/g; - s/v/\x{062B}/g; - s/j/\x{062C}/g; - s/H/\x{062D}/g; - s/x/\x{062E}/g; - s/d/\x{062F}/g; - s/\*/\x{0630}/g; - s/r/\x{0631}/g; - s/z/\x{0632}/g; - s/s/\x{0633}/g; - s/\$/\x{0634}/g; - s/S/\x{0635}/g; - s/D/\x{0636}/g; - s/T/\x{0637}/g; - s/Z/\x{0638}/g; - s/E/\x{0639}/g; - s/g/\x{063A}/g; - s/_/\x{0640}/g; - s/f/\x{0641}/g; - s/q/\x{0642}/g; - s/k/\x{0643}/g; - s/l/\x{0644}/g; - s/m/\x{0645}/g; - s/n/\x{0646}/g; - s/h/\x{0647}/g; - s/w/\x{0648}/g; - s/Y/\x{0649}/g; - s/y/\x{064A}/g; - s/F/\x{064B}/g; - s/N/\x{064C}/g; - s/K/\x{064D}/g; - s/a/\x{064E}/g; - s/u/\x{064F}/g; - s/i/\x{0650}/g; - s/\~/\x{0651}/g; - s/o/\x{0652}/g; - s/\`/\x{0670}/g; - s/\{/\x{0671}/g; - s/P/\x{067E}/g; - s/J/\x{0686}/g; - s/V/\x{06A4}/g; - s/G/\x{06AF}/g; - - -print STDOUT "$_ "; - } - print STDOUT "\n"; -} |