From 2d6a5896df99904b0c492e77168489636d545869 Mon Sep 17 00:00:00 2001 From: graehl Date: Tue, 13 Jul 2010 02:01:07 +0000 Subject: vest: combine over-similar search directions, exclude primary directions, skeleton for oracle directions git-svn-id: https://ws10smt.googlecode.com/svn/trunk@227 ec762483-ff6d-05da-a07a-a48fb63a330f --- vest/dist-vest.pl | 13 ++++- vest/line_optimizer.cc | 29 ++++++----- vest/line_optimizer.h | 4 +- vest/mr_vest_generate_mapper_input.cc | 91 +++++++++++++++++++++++++++++++---- 4 files changed, 114 insertions(+), 23 deletions(-) (limited to 'vest') diff --git a/vest/dist-vest.pl b/vest/dist-vest.pl index f8f79ee5..32a8edbd 100755 --- a/vest/dist-vest.pl +++ b/vest/dist-vest.pl @@ -49,6 +49,11 @@ my $iniFile; my $weights; my $initialWeights; my $decoderOpt; +my $noprimary; +my $maxsim=0; +my $oraclen=0; +my $oracleb=20; +my $dirargs=''; # Process command-line options Getopt::Long::Configure("no_auto_abbrev"); @@ -66,6 +71,11 @@ if (GetOptions( "normalize=s" => \$normalize, "pmem=s" => \$pmem, "rand-directions=i" => \$rand_directions, + "no-primary!" => \$noprimary, + "max-similarity=s" => \$maxsim, + "oracle-directions=i" => \$oraclen, + "oracle-batch=i" => \$oracleb, + "directions-args=s" => \$dirargs, "ref-files=s" => \$refFiles, "metric=s" => \$metric, "source-file=s" => \$srcFile, @@ -234,7 +244,8 @@ while (1){ print STDERR "\nGENERATE OPTIMIZATION STRATEGY (OPT-ITERATION $opt_iter/$optimization_iters)\n"; print STDERR `date`; $icc++; - $cmd="$MAPINPUT -w $inweights -r $dir/hgs -s $devSize -d $rand_directions > $dir/agenda.$im1-$opt_iter"; + my $nop=$noprimary?"--no_primary":""; + $cmd="$MAPINPUT -w $inweights -r $dir/hgs -s $devSize -d $rand_directions --max_similarity=$maxsim --oracle_directions=$oraclen --oracle_batch=$oracleb $dirargs > $dir/agenda.$im1-$opt_iter"; print STDERR "COMMAND:\n$cmd\n"; $result = system($cmd); unless ($result == 0){ diff --git a/vest/line_optimizer.cc b/vest/line_optimizer.cc index b6410c35..e8b40237 100644 --- a/vest/line_optimizer.cc +++ b/vest/line_optimizer.cc @@ -15,7 +15,7 @@ struct IntervalComp { bool operator() (const ErrorIter& a, const ErrorIter& b) const { return a->x < b->x; } -}; +}; double LineOptimizer::LineOptimize( const vector& surfaces, @@ -89,15 +89,22 @@ void LineOptimizer::CreateOptimizationDirections( const vector& features_to_optimize, int additional_random_directions, RandomNumberGenerator* rng, - vector >* dirs) { + vector >* dirs + , bool include_orthogonal + ) { const int num_directions = features_to_optimize.size() + additional_random_directions; - dirs->resize(num_directions); - for (int i = 0; i < num_directions; ++i) { - SparseVector& axis = (*dirs)[i]; - if (i < features_to_optimize.size()) - axis.set_value(features_to_optimize[i], 1.0); - else - RandomUnitVector(features_to_optimize, &axis, rng); - } - cerr << "Generated " << num_directions << " total axes to optimize along.\n"; + dirs->clear(); + typedef SparseVector Dir; + vector &out=*dirs; + int i=0; + if (include_orthogonal) + for (;i& primary, int additional_random_directions, RandomNumberGenerator* rng, - std::vector >* dirs); + std::vector >* dirs + , bool include_primary=true + ); }; diff --git a/vest/mr_vest_generate_mapper_input.cc b/vest/mr_vest_generate_mapper_input.cc index c96a61e4..9e702e2f 100644 --- a/vest/mr_vest_generate_mapper_input.cc +++ b/vest/mr_vest_generate_mapper_input.cc @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -7,10 +8,70 @@ #include "filelib.h" #include "weights.h" #include "line_optimizer.h" +#include "hg.h" +#include "hg_io.h" using namespace std; namespace po = boost::program_options; +typedef SparseVector Dir; + +typedef RandomNumberGenerator RNG; +RNG rng; + +struct oracle_directions { + string forest_repository; + unsigned dev_set_size; + vector dirs; //best_to_hope_dirs + vector fids; + string forest_file(unsigned i) const { + ostringstream o; + o << forest_repository << '/' << i << ".json.gz"; + return o.str(); + } + + oracle_directions(string forest_repository,unsigned dev_set_size,vector const& fids=vector()): forest_repository(forest_repository),dev_set_size(dev_set_size),fids(fids) { + dirs.resize(dev_set_size); + } + Dir const& operator[](unsigned i) { + Dir &dir=dirs[i]; + if (dir.empty()) { + ReadFile rf(forest_file(i)); + Hypergraph hg; + HypergraphIO::ReadFromJSON(rf.stream(), &hg); + cerr<<"oracle: forest["< &dirs,double min_dist,ostream *log=&cerr,bool avg=true) { + if (min_dist<=0) return; + double max_s=1.-min_dist; + unsigned N=dirs.size(); + for (int i=0;imax_s) { + if (log) *log << "Collapsing similar directions (T="< "<(),"[REQD] Current feature weights file") ("optimize_feature,o",po::value >(), "Feature to optimize (if none specified, all weights listed in the weights file will be optimized)") ("random_directions,d",po::value()->default_value(20),"Number of random directions to run the line optimizer in") + ("no_primary,n","don't use the primary (orthogonal each feature alone) directions") + ("oracle_directions,O",po::value()->default_value(0),"read the forests and choose this many directions based on heading toward a hope max (bleu+modelscore) translation.") + ("oracle_batch,b",po::value()->default_value(10),"to produce each oracle direction, sum the 'gradient' over this many sentences") + ("max_similarity,m",po::value()->default_value(0),"remove directions that are too similar (Tanimoto coeff. less than (1-this)). 0 means don't filter, 1 means only 1 direction allowed?") ("help,h", "Help"); po::options_description dcmdline_options; dcmdline_options.add(opts); @@ -43,30 +108,36 @@ void InitCommandLine(int argc, char** argv, po::variables_map* conf) { } int main(int argc, char** argv) { - RandomNumberGenerator rng; po::variables_map conf; InitCommandLine(argc, argv, &conf); Weights weights; vector features; weights.InitFromFile(conf["weights"].as(), &features); - const string forest_repository = conf["forest_repository"].as(); - assert(DirectoryExists(forest_repository)); + vector fids(features.size()); + for (int i = 0; i < features.size(); ++i) + fids[i] = FD::Convert(features[i]); + + oracle_directions od(conf["forest_repository"].as() + , conf["dev_set_size"].as() + , fids + ); +; + assert(DirectoryExists(od.forest_repository)); SparseVector origin; weights.InitSparseVector(&origin); if (conf.count("optimize_feature") > 0) features=conf["optimize_feature"].as >(); vector > axes; - vector fids(features.size()); - for (int i = 0; i < features.size(); ++i) - fids[i] = FD::Convert(features[i]); LineOptimizer::CreateOptimizationDirections( fids, conf["random_directions"].as(), &rng, - &axes); - int dev_set_size = conf["dev_set_size"].as(); - for (int i = 0; i < dev_set_size; ++i) + &axes, + !conf.count("no_primary") + ); + compress_similar(axes,conf["max_similarity"].as()); + for (int i = 0; i < od.dev_set_size; ++i) for (int j = 0; j < axes.size(); ++j) - cout << forest_repository << '/' << i << ".json.gz " << i << ' ' << origin << ' ' << axes[j] << endl; + cout << od.forest_file(i) <<" " << i << ' ' << origin << ' ' << axes[j] << endl; return 0; } -- cgit v1.2.3