From e26434979adc33bd949566ba7bf02dff64e80a3e Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Tue, 2 Oct 2012 00:19:43 -0400 Subject: cdec cleanup, remove bayesian stuff, parsing stuff --- gi/pyp-topics/src/mpi-train-contexts.cc | 201 -------------------------------- 1 file changed, 201 deletions(-) delete mode 100644 gi/pyp-topics/src/mpi-train-contexts.cc (limited to 'gi/pyp-topics/src/mpi-train-contexts.cc') diff --git a/gi/pyp-topics/src/mpi-train-contexts.cc b/gi/pyp-topics/src/mpi-train-contexts.cc deleted file mode 100644 index e05e0eac..00000000 --- a/gi/pyp-topics/src/mpi-train-contexts.cc +++ /dev/null @@ -1,201 +0,0 @@ -// STL -#include -#include -#include -#include - -// Boost -#include -#include -#include -#include -#include -#include - -// Local -#include "mpi-pyp-topics.hh" -#include "corpus.hh" -#include "mpi-corpus.hh" -#include "gzstream.hh" - -static const char *REVISION = "$Rev: 170 $"; - -// Namespaces -using namespace boost; -using namespace boost::program_options; -using namespace std; - -int main(int argc, char **argv) -{ - mpi::environment env(argc, argv); - mpi::communicator world; - int rank = world.rank(); - bool am_root = (rank==0); - if (am_root) cout << "Pitman Yor topic models: Copyright 2010 Phil Blunsom\n"; - if (am_root) std::cout << "I am process " << world.rank() << " of " << world.size() << "." << std::endl; - if (am_root) cout << REVISION << '\n' <(), "config file specifying additional command line options") - ; - options_description config_options("Allowed options"); - config_options.add_options() - ("help,h", "print help message") - ("data,d", value(), "file containing the documents and context terms") - ("topics,t", value()->default_value(50), "number of topics") - ("document-topics-out,o", value(), "file to write the document topics to") - ("default-topics-out", value(), "file to write default term topic assignments.") - ("topic-words-out,w", value(), "file to write the topic word distribution to") - ("samples,s", value()->default_value(10), "number of sampling passes through the data") - ("backoff-type", value(), "backoff type: none|simple") -// ("filter-singleton-contexts", "filter singleton contexts") - ("hierarchical-topics", "Use a backoff hierarchical PYP as the P0 for the document topics distribution.") - ("binary-counts,b", "Use binary rather than integer counts for contexts.") - ("freq-cutoff-start", value()->default_value(0), "initial frequency cutoff.") - ("freq-cutoff-end", value()->default_value(0), "final frequency cutoff.") - ("freq-cutoff-interval", value()->default_value(0), "number of iterations between frequency decrement.") - ("max-contexts-per-document", value()->default_value(0), "Only sample the n most frequent contexts for a document.") - ; - - cmdline_specific.add(config_options); - - store(parse_command_line(argc, argv, cmdline_specific), vm); - notify(vm); - - if (vm.count("config") > 0) { - ifstream config(vm["config"].as().c_str()); - store(parse_config_file(config, config_options), vm); - } - - if (vm.count("help")) { - cout << cmdline_specific << "\n"; - return 1; - } - } - //////////////////////////////////////////////////////////////////////////////////////////// - - if (!vm.count("data")) { - cerr << "Please specify a file containing the data." << endl; - return 1; - } - - // seed the random number generator: 0 = automatic, specify value otherwise - unsigned long seed = 0; - MPIPYPTopics model(vm["topics"].as(), vm.count("hierarchical-topics"), seed); - - // read the data - BackoffGenerator* backoff_gen=0; - if (vm.count("backoff-type")) { - if (vm["backoff-type"].as() == "none") { - backoff_gen = 0; - } - else if (vm["backoff-type"].as() == "simple") { - backoff_gen = new SimpleBackoffGenerator(); - } - else { - cerr << "Backoff type (--backoff-type) must be one of none|simple." <(), backoff_gen, /*vm.count("filter-singleton-contexts")*/ false, vm.count("binary-counts")); - int mpi_start = 0, mpi_end = 0; - contexts_corpus.bounds(&mpi_start, &mpi_end); - std::cerr << "\tProcess " << rank << " has documents " << mpi_start << " -> " << mpi_end << "." << std::endl; - - model.set_backoff(contexts_corpus.backoff_index()); - - if (backoff_gen) - delete backoff_gen; - - // train the sampler - model.sample_corpus(contexts_corpus, vm["samples"].as(), - vm["freq-cutoff-start"].as(), - vm["freq-cutoff-end"].as(), - vm["freq-cutoff-interval"].as(), - vm["max-contexts-per-document"].as()); - - if (vm.count("document-topics-out")) { - std::ofstream documents_out((vm["document-topics-out"].as() + ".pyp-process-" + boost::lexical_cast(rank)).c_str()); - //int documents = contexts_corpus.num_documents(); - /* - int mpi_start = 0, mpi_end = documents; - if (world.size() != 1) { - mpi_start = (documents / world.size()) * rank; - if (rank == world.size()-1) mpi_end = documents; - else mpi_end = (documents / world.size())*(rank+1); - } - */ - - map all_terms; - for (int document_id=mpi_start; document_id unique_terms; - for (Document::const_iterator docIt=doc.begin(); docIt != doc.end(); ++docIt) { - if (unique_terms.empty() || *docIt != unique_terms.back()) - unique_terms.push_back(*docIt); - // increment this terms frequency - pair::iterator,bool> insert_result = all_terms.insert(make_pair(*docIt,1)); - if (!insert_result.second) - all_terms[*docIt] = all_terms[*docIt] + 1; - } - documents_out << contexts_corpus.key(document_id) << '\t'; - documents_out << model.max(document_id).first << " " << doc.size() << " ||| "; - for (std::vector::const_iterator termIt=unique_terms.begin(); termIt != unique_terms.end(); ++termIt) { - if (termIt != unique_terms.begin()) - documents_out << " ||| "; - vector strings = contexts_corpus.context2string(*termIt); - copy(strings.begin(), strings.end(),ostream_iterator(documents_out, " ")); - std::pair maxinfo = model.max(document_id, *termIt); - documents_out << "||| C=" << maxinfo.first << " P=" << maxinfo.second; - } - documents_out <().c_str()); - for (int p=0; p < world.size(); ++p) { - std::string rank_p_prefix((vm["document-topics-out"].as() + ".pyp-process-" + boost::lexical_cast(p)).c_str()); - std::ifstream rank_p_trees_istream(rank_p_prefix.c_str(), std::ios_base::binary); - root_documents_out << rank_p_trees_istream.rdbuf(); - rank_p_trees_istream.close(); - remove((rank_p_prefix).c_str()); - } - root_documents_out.close(); - } - - if (am_root && vm.count("default-topics-out")) { - ofstream default_topics(vm["default-topics-out"].as().c_str()); - default_topics << model.max_topic() <::const_iterator termIt=all_terms.begin(); termIt != all_terms.end(); ++termIt) { - vector strings = contexts_corpus.context2string(termIt->first); - default_topics << model.max(-1, termIt->first).first << " ||| " << termIt->second << " ||| "; - copy(strings.begin(), strings.end(),ostream_iterator(default_topics, " ")); - default_topics <().c_str()); - model.print_topic_terms(topics_out); - topics_out.close(); - } - - cout <