From 39e4369f2d6fee603cc7b1bcb92b6ed341fd31a3 Mon Sep 17 00:00:00 2001 From: Jonathan Clark Date: Wed, 23 May 2012 13:48:04 -0400 Subject: Script for dividing a single file full of references into the 1-reference-per-file format expected by cdec --- dpmert/divide_refs.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 dpmert/divide_refs.py (limited to 'dpmert') diff --git a/dpmert/divide_refs.py b/dpmert/divide_refs.py new file mode 100755 index 00000000..b478f918 --- /dev/null +++ b/dpmert/divide_refs.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import sys + +(numRefs, outPrefix) = sys.argv[1:] +numRefs = int(numRefs) + +outs = [open(outPrefix+str(i), "w") for i in range(numRefs)] + +i = 0 +for line in sys.stdin: + outs[i].write(line) + i = (i + 1) % numRefs + +for out in outs: + out.close() -- cgit v1.2.3 From 570ba076cbe3b12c56b281da7c1892972e8598f1 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Wed, 23 May 2012 18:02:48 -0400 Subject: more bjam stuff, more cleanup --- Jamroot | 4 ++-- creg/README | 3 +++ creg/test_data/shuttle.testfeat | 1 + creg/test_data/shuttle.testresp | 1 + creg/test_data/shuttle.trainfeat | 23 +++++++++++++++++++++++ creg/test_data/shuttle.trainresp | 23 +++++++++++++++++++++++ decoder/bottom_up_parser.cc | 24 ++++++++++++------------ decoder/ff_dwarf.cc | 6 +++--- decoder/hg.h | 2 +- decoder/hg_io.cc | 9 ++++----- decoder/inside_outside.h | 10 +++++----- decoder/kbest.h | 22 +++++++++++++--------- decoder/trule.cc | 24 ++++++++++++------------ dpmert/Jamfile | 30 ++++++++++++++++++++++++++++++ dpmert/ces.cc | 2 +- dpmert/lo_test.cc | 4 ++-- dpmert/mr_dpmert_generate_mapper_input.cc | 7 +++++-- dpmert/mr_dpmert_map.cc | 2 +- pro-train/Makefile.am | 2 +- pro-train/mr_pro_reduce.cc | 9 ++++++--- utils/Jamfile | 4 +--- utils/null_traits.h | 8 ++++---- utils/stringlib.h | 14 +++++++------- 23 files changed, 161 insertions(+), 73 deletions(-) create mode 100644 creg/test_data/shuttle.testfeat create mode 100644 creg/test_data/shuttle.testresp create mode 100644 creg/test_data/shuttle.trainfeat create mode 100644 creg/test_data/shuttle.trainresp create mode 100644 dpmert/Jamfile (limited to 'dpmert') diff --git a/Jamroot b/Jamroot index 8be59e5a..edc28e23 100644 --- a/Jamroot +++ b/Jamroot @@ -30,9 +30,9 @@ if [ test_header boost/serialization/map.hpp ] && [ test_library boost_serializa project : requirements $(requirements) darwin:static ; project : default-build single on release ; -install-bin-libs utils//programs mteval//programs klm/lm//programs training//liblbfgs decoder//cdec creg//creg phrasinator//programs mira//kbest_mira ; +install-bin-libs dpmert//programs utils//programs mteval//programs klm/lm//programs training//liblbfgs decoder//cdec creg//creg phrasinator//programs mira//kbest_mira ; -build-projects mteval decoder klm/lm training/liblbfgs creg ; +build-projects mteval decoder dpmert klm/lm training/liblbfgs creg ; #Compile everything ending with _test.cc into a test and run it. rule all_tests ( targets * : dependencies : properties * ) { diff --git a/creg/README b/creg/README index 28a71d11..208cabf5 100644 --- a/creg/README +++ b/creg/README @@ -15,3 +15,6 @@ Linear regression example (training and testing): $ ./creg -n -x test_data/auto-mpg.trainfeat -y test_data/auto-mpg.trainresp --l2 1000 \ -t test_data/auto-mpg.testfeat -s test_data/auto-mpg.testresp > weights.txt +Ordinal regression example + $ ./creg -o -x test_data/shuttle.trainfeat -y test_data/shuttle.trainresp + diff --git a/creg/test_data/shuttle.testfeat b/creg/test_data/shuttle.testfeat new file mode 100644 index 00000000..31c07017 --- /dev/null +++ b/creg/test_data/shuttle.testfeat @@ -0,0 +1 @@ +Challenger {"temp":31, "date":9524} diff --git a/creg/test_data/shuttle.testresp b/creg/test_data/shuttle.testresp new file mode 100644 index 00000000..249e84c7 --- /dev/null +++ b/creg/test_data/shuttle.testresp @@ -0,0 +1 @@ +Challenger 2 diff --git a/creg/test_data/shuttle.trainfeat b/creg/test_data/shuttle.trainfeat new file mode 100644 index 00000000..c17d0f7f --- /dev/null +++ b/creg/test_data/shuttle.trainfeat @@ -0,0 +1,23 @@ +1 {"temp":66, "date":7772} +2 {"temp":70, "date":7986} +3 {"temp":69, "date":8116} +5 {"temp":68, "date":8350} +6 {"temp":67, "date":8494} +7 {"temp":72, "date":8569} +8 {"temp":73, "date":8642} +9 {"temp":70, "date":8732} +10 {"temp":57, "date":8799} +11 {"temp":63, "date":8862} +12 {"temp":70, "date":9008} +13 {"temp":78, "date":9044} +14 {"temp":67, "date":9078} +15 {"temp":53, "date":9155} +16 {"temp":67, "date":9233} +17 {"temp":75, "date":9250} +18 {"temp":70, "date":9299} +19 {"temp":81, "date":9341} +20 {"temp":76, "date":9370} +21 {"temp":79, "date":9407} +22 {"temp":75, "date":9434} +23 {"temp":76, "date":9461} +24 {"temp":58, "date":9508} diff --git a/creg/test_data/shuttle.trainresp b/creg/test_data/shuttle.trainresp new file mode 100644 index 00000000..53e58f2c --- /dev/null +++ b/creg/test_data/shuttle.trainresp @@ -0,0 +1,23 @@ +1 0 +2 1 +3 0 +5 0 +6 1 +7 0 +8 0 +9 0 +10 1 +11 2 +12 2 +13 0 +14 0 +15 2 +16 2 +17 2 +18 2 +19 1 +20 1 +21 0 +22 2 +23 1 +24 2 diff --git a/decoder/bottom_up_parser.cc b/decoder/bottom_up_parser.cc index 63939221..1f262747 100644 --- a/decoder/bottom_up_parser.cc +++ b/decoder/bottom_up_parser.cc @@ -154,7 +154,7 @@ PassiveChart::PassiveChart(const string& goal, goal_idx_(-1), lc_fid_(FD::Convert("LatticeCost")) { act_chart_.resize(grammars_.size()); - for (int i = 0; i < grammars_.size(); ++i) + for (unsigned i = 0; i < grammars_.size(); ++i) act_chart_[i] = new ActiveChart(forest, *this); if (!kGOAL) kGOAL = TD::Convert("Goal") * -1; if (!SILENT) cerr << " Goal category: [" << goal << ']' << endl; @@ -204,12 +204,12 @@ void PassiveChart::ApplyRules(const int i, void PassiveChart::ApplyUnaryRules(const int i, const int j) { const vector& nodes = chart_(i,j); // reference is important! - for (int gi = 0; gi < grammars_.size(); ++gi) { + for (unsigned gi = 0; gi < grammars_.size(); ++gi) { if (!grammars_[gi]->HasRuleForSpan(i,j,input_.Distance(i,j))) continue; - for (int di = 0; di < nodes.size(); ++di) { + for (unsigned di = 0; di < nodes.size(); ++di) { const WordID& cat = forest_->nodes_[nodes[di]].cat_; const vector& unaries = grammars_[gi]->GetUnaryRulesForRHS(cat); - for (int ri = 0; ri < unaries.size(); ++ri) { + for (unsigned ri = 0; ri < unaries.size(); ++ri) { // cerr << "At (" << i << "," << j << "): applying " << unaries[ri]->AsString() << endl; const Hypergraph::TailNodeVector ant(1, nodes[di]); ApplyRule(i, j, unaries[ri], ant, 0); // may update nodes @@ -224,15 +224,15 @@ bool PassiveChart::Parse() { size_t res = min(static_cast(2000000), static_cast(in_size_2 * 1000)); forest_->edges_.reserve(res); goal_idx_ = -1; - for (int gi = 0; gi < grammars_.size(); ++gi) + for (unsigned gi = 0; gi < grammars_.size(); ++gi) act_chart_[gi]->SeedActiveChart(*grammars_[gi]); if (!SILENT) cerr << " "; - for (int l=1; lAdvanceDotsForAllItemsInCell(i, j, input_); @@ -248,7 +248,7 @@ bool PassiveChart::Parse() { } ApplyUnaryRules(i,j); - for (int gi = 0; gi < grammars_.size(); ++gi) { + for (unsigned gi = 0; gi < grammars_.size(); ++gi) { const Grammar& g = *grammars_[gi]; // deal with non-terminals that were just proved if (g.HasRuleForSpan(i, j, input_.Distance(i,j))) @@ -256,7 +256,7 @@ bool PassiveChart::Parse() { } } const vector& dh = chart_(0, input_.size()); - for (int di = 0; di < dh.size(); ++di) { + for (unsigned di = 0; di < dh.size(); ++di) { const Hypergraph::Node& node = forest_->nodes_[dh[di]]; if (node.cat_ == goal_cat_) { Hypergraph::TailNodeVector ant(1, node.id_); @@ -272,7 +272,7 @@ bool PassiveChart::Parse() { } PassiveChart::~PassiveChart() { - for (int i = 0; i < act_chart_.size(); ++i) + for (unsigned i = 0; i < act_chart_.size(); ++i) delete act_chart_[i]; } diff --git a/decoder/ff_dwarf.cc b/decoder/ff_dwarf.cc index 3daa85ac..43528405 100644 --- a/decoder/ff_dwarf.cc +++ b/decoder/ff_dwarf.cc @@ -519,7 +519,7 @@ void Dwarf::neighboringFWs(const Lattice& l, const int& i, const int& j, const m while (idx>=0) { if (l[idx].size()>0) { if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) { - *lfw++; + lfw++; } } idx-=l[idx][0].dist2next; @@ -528,7 +528,7 @@ void Dwarf::neighboringFWs(const Lattice& l, const int& i, const int& j, const m while (idx0) { if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) { - *rfw++; + rfw++; } } idx+=l[idx][0].dist2next; @@ -787,7 +787,7 @@ bool Dwarf::generalizeOrientation(CountTable* table, const std::map void Reweight(const V& weights) { - for (int i = 0; i < edges_.size(); ++i) { + for (unsigned i = 0; i < edges_.size(); ++i) { Edge& e = edges_[i]; e.edge_prob_.logeq(e.feature_values_.dot(weights)); } diff --git a/decoder/hg_io.cc b/decoder/hg_io.cc index 734c2ce8..3321558d 100644 --- a/decoder/hg_io.cc +++ b/decoder/hg_io.cc @@ -488,13 +488,13 @@ int getInt(const std::string& in, int &c) #define MAX_NODES 100000000 // parse ('foo', 0.23) void ReadPLFEdge(const std::string& in, int &c, int cur_node, Hypergraph* hg) { - if (get(in,c++) != '(') { assert(!"PCN/PLF parse error: expected ( at start of cn alt block\n"); } + if (get(in,c++) != '(') { cerr << "PCN/PLF parse error: expected (\n"; abort(); } vector ewords(2, 0); ewords[1] = TD::Convert(getEscapedString(in,c)); TRulePtr r(new TRule(ewords)); r->ComputeArity(); // cerr << "RULE: " << r->AsString() << endl; - if (get(in,c++) != ',') { cerr << in << endl; assert(!"PCN/PLF parse error: expected , after string\n"); } + if (get(in,c++) != ',') { cerr << in << endl; cerr << "PCN/PLF parse error: expected , after string\n"; abort(); } size_t cnNext = 1; std::vector probs; probs.push_back(getFloat(in,c)); @@ -508,10 +508,9 @@ void ReadPLFEdge(const std::string& in, int &c, int cur_node, Hypergraph* hg) { if (probs.size()>1) { cnNext = static_cast(probs.back()); probs.pop_back(); - if (cnNext < 1) { cerr << cnNext << endl; - assert(!"PCN/PLF parse error: bad link length at last element of cn alt block\n"); } + if (cnNext < 1) { cerr << cnNext << endl << "PCN/PLF parse error: bad link length at last element of cn alt block\n"; abort(); } } - if (get(in,c++) != ')') { assert(!"PCN/PLF parse error: expected ) at end of cn alt block\n"); } + if (get(in,c++) != ')') { cerr << "PCN/PLF parse error: expected ) at end of cn alt block\n"; abort(); } eatws(in,c); Hypergraph::TailNodeVector tail(1, cur_node); Hypergraph::Edge* edge = hg->AddEdge(r, tail); diff --git a/decoder/inside_outside.h b/decoder/inside_outside.h index dc96f1a9..2ded328d 100644 --- a/decoder/inside_outside.h +++ b/decoder/inside_outside.h @@ -31,24 +31,24 @@ template WeightType Inside(const Hypergraph& hg, std::vector* result = NULL, const WeightFunction& weight = WeightFunction()) { - const int num_nodes = hg.nodes_.size(); + const unsigned num_nodes = hg.nodes_.size(); std::vector dummy; std::vector& inside_score = result ? *result : dummy; inside_score.clear(); inside_score.resize(num_nodes); // std::fill(inside_score.begin(), inside_score.end(), WeightType()); // clear handles - for (int i = 0; i < num_nodes; ++i) { + for (unsigned i = 0; i < num_nodes; ++i) { WeightType* const cur_node_inside_score = &inside_score[i]; Hypergraph::EdgesVector const& in=hg.nodes_[i].in_edges_; - const int num_in_edges = in.size(); + const unsigned num_in_edges = in.size(); if (num_in_edges == 0) { *cur_node_inside_score = WeightType(1); //FIXME: why not call weight(edge) instead? continue; } - for (int j = 0; j < num_in_edges; ++j) { + for (unsigned j = 0; j < num_in_edges; ++j) { const Hypergraph::Edge& edge = hg.edges_[in[j]]; WeightType score = weight(edge); - for (int k = 0; k < edge.tail_nodes_.size(); ++k) { + for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) { const int tail_node_index = edge.tail_nodes_[k]; score *= inside_score[tail_node_index]; } diff --git a/decoder/kbest.h b/decoder/kbest.h index 03a8311c..9af3a20e 100644 --- a/decoder/kbest.h +++ b/decoder/kbest.h @@ -43,7 +43,7 @@ namespace KBest { traverse(tf), w(wf), g(hg), nds(g.nodes_.size()), k_prime(k) {} ~KBestDerivations() { - for (int i = 0; i < freelist.size(); ++i) + for (unsigned i = 0; i < freelist.size(); ++i) delete freelist[i]; } @@ -86,7 +86,7 @@ namespace KBest { // Hypergraph::Edge const * operator ->() const { return d->edge; } }; - EdgeHandle operator()(int t,int taili,EdgeHandle const& parent) const { + EdgeHandle operator()(unsigned t,unsigned taili,EdgeHandle const& parent) const { return EdgeHandle(nds[t].D[parent.d->j[taili]]); } @@ -98,7 +98,7 @@ namespace KBest { size_t operator()(const Derivation* d) const { size_t x = 5381; x = ((x << 5) + x) ^ d->edge->id_; - for (int i = 0; i < d->j.size(); ++i) + for (unsigned i = 0; i < d->j.size(); ++i) x = ((x << 5) + x) ^ d->j[i]; return x; } @@ -121,7 +121,7 @@ namespace KBest { explicit NodeDerivationState(const DerivationFilter& f = DerivationFilter()) : filter(f) {} }; - Derivation* LazyKthBest(int v, int k) { + Derivation* LazyKthBest(unsigned v, unsigned k) { NodeDerivationState& s = GetCandidates(v); CandidateHeap& cand = s.cand; DerivationList& D = s.D; @@ -139,7 +139,7 @@ namespace KBest { Derivation* d = cand.back(); cand.pop_back(); std::vector ants(d->edge->Arity()); - for (int j = 0; j < ants.size(); ++j) + for (unsigned j = 0; j < ants.size(); ++j) ants[j] = &LazyKthBest(d->edge->tail_nodes_[j], d->j[j])->yield; traverse(*d->edge, ants, &d->yield); if (!filter(d->yield)) { @@ -171,12 +171,12 @@ namespace KBest { return freelist.back(); } - NodeDerivationState& GetCandidates(int v) { + NodeDerivationState& GetCandidates(unsigned v) { NodeDerivationState& s = nds[v]; if (!s.D.empty() || !s.cand.empty()) return s; const Hypergraph::Node& node = g.nodes_[v]; - for (int i = 0; i < node.in_edges_.size(); ++i) { + for (unsigned i = 0; i < node.in_edges_.size(); ++i) { const Hypergraph::Edge& edge = g.edges_[node.in_edges_[i]]; SmallVectorInt jv(edge.Arity(), 0); Derivation* d = CreateDerivation(edge, jv); @@ -184,7 +184,7 @@ namespace KBest { s.cand.push_back(d); } - const int effective_k = std::min(k_prime, s.cand.size()); + const unsigned effective_k = std::min(k_prime, s.cand.size()); const typename CandidateHeap::iterator kth = s.cand.begin() + effective_k; std::nth_element(s.cand.begin(), kth, s.cand.end(), DerivationCompare()); s.cand.resize(effective_k); @@ -194,7 +194,7 @@ namespace KBest { } void LazyNext(const Derivation* d, CandidateHeap* cand, UniqueDerivationSet* ds) { - for (int i = 0; i < d->j.size(); ++i) { + for (unsigned i = 0; i < d->j.size(); ++i) { SmallVectorInt j = d->j; ++j[i]; const Derivation* ant = LazyKthBest(d->edge->tail_nodes_[i], j[i]); @@ -205,8 +205,12 @@ namespace KBest { if (new_d) { cand->push_back(new_d); std::push_heap(cand->begin(), cand->end(), HeapCompare()); +#ifdef NDEBUG + ds->insert(new_d).second; // insert into uniqueness set +#else bool inserted = ds->insert(new_d).second; // insert into uniqueness set assert(inserted); +#endif } } } diff --git a/decoder/trule.cc b/decoder/trule.cc index 141b8faa..5ebc4c16 100644 --- a/decoder/trule.cc +++ b/decoder/trule.cc @@ -18,7 +18,7 @@ bool TRule::IsGoal() const { } static WordID ConvertTrgString(const string& w) { - int len = w.size(); + const unsigned len = w.size(); WordID id = 0; // [X,0] or [0] // for target rules, we ignore the category, just keep the index @@ -33,7 +33,7 @@ static WordID ConvertTrgString(const string& w) { } static WordID ConvertSrcString(const string& w, bool mono = false) { - int len = w.size(); + const unsigned len = w.size(); // [X,0] // for source rules, we keep the category and ignore the index (source rules are // always numbered 1, 2, 3... @@ -60,7 +60,7 @@ static WordID ConvertSrcString(const string& w, bool mono = false) { static WordID ConvertLHS(const string& w) { if (w[0] == '[') { - int len = w.size(); + const unsigned len = w.size(); if (len < 3) { cerr << "Format error: " << w << endl; exit(1); } return TD::Convert(w.substr(1, len-2)) * -1; } else { @@ -143,15 +143,15 @@ bool TRule::ReadFromString(const string& line, bool strict, bool mono) { string ss; getline(is, ss); //cerr << "L: " << ss << endl; - int start = 0; - int len = ss.size(); + unsigned start = 0; + unsigned len = ss.size(); const size_t ppos = ss.find(" |||"); if (ppos != string::npos) { len = ppos; } while (start < len) { while(start < len && (ss[start] == ' ' || ss[start] == ';')) ++start; if (start == len) break; - int end = start + 1; + unsigned end = start + 1; while(end < len && (ss[end] != '=' && ss[end] != ' ' && ss[end] != ';')) ++end; if (end == len || ss[end] == ' ' || ss[end] == ';') { @@ -188,7 +188,7 @@ bool TRule::ReadFromString(const string& line, bool strict, bool mono) { while(is>>w && w!="|||") { e_.push_back(ConvertTrgString(w)); } f_ = e_; int x = ConvertLHS("[X]"); - for (int i = 0; i < f_.size(); ++i) + for (unsigned i = 0; i < f_.size(); ++i) if (f_[i] <= 0) { f_[i] = x; } } else { cerr << "F: " << format << endl; @@ -197,7 +197,7 @@ bool TRule::ReadFromString(const string& line, bool strict, bool mono) { if (mono) { e_ = f_; int ci = 0; - for (int i = 0; i < e_.size(); ++i) + for (unsigned i = 0; i < e_.size(); ++i) if (e_[i] < 0) e_[i] = ci--; } @@ -208,7 +208,7 @@ bool TRule::ReadFromString(const string& line, bool strict, bool mono) { bool TRule::SanityCheck() const { vector used(f_.size(), 0); int ac = 0; - for (int i = 0; i < e_.size(); ++i) { + for (unsigned i = 0; i < e_.size(); ++i) { int ind = e_[i]; if (ind > 0) continue; ind = -ind; @@ -238,7 +238,7 @@ string TRule::AsString(bool verbose) const { if (lhs_ && verbose) { os << '[' << TD::Convert(lhs_ * -1) << "] |||"; } - for (int i = 0; i < f_.size(); ++i) { + for (unsigned i = 0; i < f_.size(); ++i) { const WordID& w = f_[i]; if (w < 0) { int wi = w * -1; @@ -249,7 +249,7 @@ string TRule::AsString(bool verbose) const { } } os << " ||| "; - for (int i =0; i. + : : + ..//utils + ..//mteval + ../klm/lm//kenlm + ..//boost_program_options + . + ; + +all_tests [ glob *_test.cc ] : dpmert : $(TOP)/dpmert/test_data ; + +exe sentserver : sentserver.c ; +exe sentclient : sentclient.c ; +exe mr_dpmert_generate_mapper_input : mr_dpmert_generate_mapper_input.cc dpmert ..//boost_program_options ; +exe mr_dpmert_map : mr_dpmert_map.cc dpmert ..//boost_program_options ; +exe mr_dpmert_reduce : mr_dpmert_reduce.cc dpmert ..//boost_program_options ; diff --git a/dpmert/ces.cc b/dpmert/ces.cc index c6cb1cdf..157b2d17 100644 --- a/dpmert/ces.cc +++ b/dpmert/ces.cc @@ -25,7 +25,7 @@ void ComputeErrorSurface(const SegmentEvaluator& ss, env->resize(ienv.size()); SufficientStats prev_score; // defaults to 0 int j = 0; - for (int i = 0; i < ienv.size(); ++i) { + for (unsigned i = 0; i < ienv.size(); ++i) { const MERTPoint& seg = *ienv[i]; vector trans; #if 0 diff --git a/dpmert/lo_test.cc b/dpmert/lo_test.cc index d47a95b5..f6df523e 100644 --- a/dpmert/lo_test.cc +++ b/dpmert/lo_test.cc @@ -88,7 +88,7 @@ BOOST_AUTO_TEST_CASE(TestConvexHullInside) { if (!d) break; cerr << log(d->score) << " ||| " << TD::GetString(d->yield) << " ||| " << d->feature_values << endl; } - for (int i = 0; i < segs.size(); ++i) { + for (unsigned i = 0; i < segs.size(); ++i) { cerr << "seg=" << i << endl; vector trans; segs[i]->ConstructTranslation(&trans); @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE( TestS1) { &rng, &axes); assert(axes.size() == 10 + to_optimize.size()); - for (int i = 0; i < axes.size(); ++i) + for (unsigned i = 0; i < axes.size(); ++i) cerr << axes[i] << endl; const SparseVector& axis = axes[0]; diff --git a/dpmert/mr_dpmert_generate_mapper_input.cc b/dpmert/mr_dpmert_generate_mapper_input.cc index 59d4f24f..199cd23a 100644 --- a/dpmert/mr_dpmert_generate_mapper_input.cc +++ b/dpmert/mr_dpmert_generate_mapper_input.cc @@ -52,12 +52,15 @@ int main(int argc, char** argv) { Weights::InitFromFile(conf["weights"].as(), &w, &features); Weights::InitSparseVector(w, &origin); const string forest_repository = conf["forest_repository"].as(); - assert(DirectoryExists(forest_repository)); + if (!DirectoryExists(forest_repository)) { + cerr << "Forest repository directory " << forest_repository << " not found!\n"; + return 1; + } if (conf.count("optimize_feature") > 0) features=conf["optimize_feature"].as >(); vector > directions; vector fids(features.size()); - for (int i = 0; i < features.size(); ++i) + for (unsigned i = 0; i < features.size(); ++i) fids[i] = FD::Convert(features[i]); LineOptimizer::CreateOptimizationDirections( fids, diff --git a/dpmert/mr_dpmert_map.cc b/dpmert/mr_dpmert_map.cc index f3304f0f..d1efcf96 100644 --- a/dpmert/mr_dpmert_map.cc +++ b/dpmert/mr_dpmert_map.cc @@ -52,7 +52,7 @@ bool ReadSparseVectorString(const string& s, SparseVector* v) { vector fields; Tokenize(s, ';', &fields); if (fields.empty()) return false; - for (int i = 0; i < fields.size(); ++i) { + for (unsigned i = 0; i < fields.size(); ++i) { vector pair(2); Tokenize(fields[i], '=', &pair); if (pair.size() != 2) { diff --git a/pro-train/Makefile.am b/pro-train/Makefile.am index 11d26211..a98dd245 100644 --- a/pro-train/Makefile.am +++ b/pro-train/Makefile.am @@ -8,6 +8,6 @@ mr_pro_map_SOURCES = mr_pro_map.cc mr_pro_map_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lz mr_pro_reduce_SOURCES = mr_pro_reduce.cc -mr_pro_reduce_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/training/optimize.o $(top_srcdir)/training/liblbfgs/liblbfgs.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lz +mr_pro_reduce_LDADD = $(top_srcdir)/training/liblbfgs/liblbfgs.a $(top_srcdir)/utils/libutils.a -lz AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(GTEST_CPPFLAGS) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training diff --git a/pro-train/mr_pro_reduce.cc b/pro-train/mr_pro_reduce.cc index d3fb8026..9698bb5d 100644 --- a/pro-train/mr_pro_reduce.cc +++ b/pro-train/mr_pro_reduce.cc @@ -25,6 +25,7 @@ void InitCommandLine(int argc, char** argv, po::variables_map* conf) { opts.add_options() ("weights,w", po::value(), "Weights from previous iteration (used as initialization and interpolation") ("regularization_strength,C",po::value()->default_value(500.0), "l2 regularization strength") + ("l1",po::value()->default_value(0.0), "l1 regularization strength") ("regularize_to_weights,y",po::value()->default_value(5000.0), "Differences in learned weights to previous weights are penalized with an l2 penalty with this strength; 0.0 = no effect") ("memory_buffers,m",po::value()->default_value(100), "Number of memory buffers (LBFGS)") ("min_reg,r",po::value()->default_value(0.01), "When tuning (-T) regularization strength, minimum regularization strenght") @@ -180,12 +181,13 @@ struct ProLoss { double LearnParameters(const vector > >& training, const vector > >& testing, const double C, + const double C1, const double T, const unsigned memory_buffers, const vector& prev_x, vector* px) { ProLoss loss(training, testing, C, T, prev_x); - LBFGS lbfgs(px, loss, 0.0, memory_buffers); + LBFGS lbfgs(px, loss, C1, memory_buffers); lbfgs.MinimizeFunction(); return loss.tppl; } @@ -203,6 +205,7 @@ int main(int argc, char** argv) { const double min_reg = conf["min_reg"].as(); const double max_reg = conf["max_reg"].as(); double C = conf["regularization_strength"].as(); // will be overridden if parameter is tuned + double C1 = conf["l1"].as(); // will be overridden if parameter is tuned const double T = conf["regularize_to_weights"].as(); assert(C >= 0.0); assert(min_reg >= 0.0); @@ -239,7 +242,7 @@ int main(int argc, char** argv) { cerr << "SWEEP FACTOR: " << sweep_factor << endl; while(C < max_reg) { cerr << "C=" << C << "\tT=" <(), prev_x, &x); + tppl = LearnParameters(training, testing, C, C1, T, conf["memory_buffers"].as(), prev_x, &x); sp.push_back(make_pair(C, tppl)); C *= sweep_factor; } @@ -262,7 +265,7 @@ int main(int argc, char** argv) { } C = sp[best_i].first; } // tune regularizer - tppl = LearnParameters(training, testing, C, T, conf["memory_buffers"].as(), prev_x, &x); + tppl = LearnParameters(training, testing, C, C1, T, conf["memory_buffers"].as(), prev_x, &x); if (conf.count("weights")) { for (int i = 1; i < x.size(); ++i) { x[i] = (x[i] * psi) + prev_x[i] * (1.0 - psi); diff --git a/utils/Jamfile b/utils/Jamfile index 53a51277..4444b25f 100644 --- a/utils/Jamfile +++ b/utils/Jamfile @@ -24,11 +24,9 @@ lib utils : ..//z : .. . : : .. . ; -exe ts : ts.cc utils ; exe atools : atools.cc utils ..//boost_program_options ; -exe phmt : phmt.cc utils ; exe reconstruct_weights : reconstruct_weights.cc utils ..//boost_program_options ; alias programs : reconstruct_weights atools ; -all_tests : utils ; +all_tests [ glob *_test.cc phmt.cc ts.cc ] : utils : $(TOP)/utils/test_data ; diff --git a/utils/null_traits.h b/utils/null_traits.h index fac857d9..7b2d32d0 100644 --- a/utils/null_traits.h +++ b/utils/null_traits.h @@ -3,23 +3,23 @@ template struct null_traits { - static V null; //TODO: maybe take out default null and make ppl explicitly define? they may be surprised that they need to when they include a header lib that uses null_traits + static V xnull; //TODO: maybe take out default null and make ppl explicitly define? they may be surprised that they need to when they include a header lib that uses null_traits }; // global bool is_null(V const& v) // definitely override this, and possibly set_null and is_null. that's the point. template -V null_traits::null; +V null_traits::xnull; //TODO: are we getting single init of the static null object? template void set_null(V &v) { - v=null_traits::null; + v=null_traits::xnull; } template void is_null(V const& v) { - return v==null_traits::null; + return v==null_traits::xnull; } diff --git a/utils/stringlib.h b/utils/stringlib.h index f457e1e4..13d14dbf 100644 --- a/utils/stringlib.h +++ b/utils/stringlib.h @@ -97,9 +97,9 @@ inline std::string Trim(const std::string& str, const std::string& dropChars = " inline void Tokenize(const std::string& str, char delimiter, std::vector* res) { std::string s = str; - int last = 0; + unsigned last = 0; res->clear(); - for (int i=0; i < s.size(); ++i) + for (unsigned i=0; i < s.size(); ++i) if (s[i] == delimiter) { s[i]=0; if (last != i) { @@ -120,14 +120,14 @@ inline unsigned NTokens(const std::string& str, char delimiter) inline std::string LowercaseString(const std::string& in) { std::string res(in.size(),' '); - for (int i = 0; i < in.size(); ++i) + for (unsigned i = 0; i < in.size(); ++i) res[i] = tolower(in[i]); return res; } inline std::string UppercaseString(const std::string& in) { std::string res(in.size(),' '); - for (int i = 0; i < in.size(); ++i) + for (unsigned i = 0; i < in.size(); ++i) res[i] = toupper(in[i]); return res; } @@ -146,8 +146,8 @@ inline int CountSubstrings(const std::string& str, const std::string& sub) { inline int SplitOnWhitespace(const std::string& in, std::vector* out) { out->clear(); - int i = 0; - int start = 0; + unsigned i = 0; + unsigned start = 0; std::string cur; while(i < in.size()) { if (in[i] == ' ' || in[i] == '\t') { @@ -249,7 +249,7 @@ inline void SplitCommandAndParam(const std::string& in, std::string* cmd, std::s SplitOnWhitespace(in, &x); if (x.size() == 0) return; *cmd = x[0]; - for (int i = 1; i < x.size(); ++i) { + for (unsigned i = 1; i < x.size(); ++i) { if (i > 1) { *param += " "; } *param += x[i]; } -- cgit v1.2.3 From 42326517b69ed772bb6c9f1d6b166176cc9ddf23 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Wed, 23 May 2012 18:35:03 -0400 Subject: dpmert jam --- dpmert/Jamfile | 2 ++ 1 file changed, 2 insertions(+) (limited to 'dpmert') diff --git a/dpmert/Jamfile b/dpmert/Jamfile index d96c592c..2bbf4f18 100644 --- a/dpmert/Jamfile +++ b/dpmert/Jamfile @@ -28,3 +28,5 @@ exe sentclient : sentclient.c ; exe mr_dpmert_generate_mapper_input : mr_dpmert_generate_mapper_input.cc dpmert ..//boost_program_options ; exe mr_dpmert_map : mr_dpmert_map.cc dpmert ..//boost_program_options ; exe mr_dpmert_reduce : mr_dpmert_reduce.cc dpmert ..//boost_program_options ; + +alias programs : sentserver sentclient mr_dpmert_generate_mapper_input mr_dpmert_map mr_dpmert_reduce ; -- cgit v1.2.3 From a0b6d78b597b8620e934e02092ce0b9c20204d8f Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Fri, 25 May 2012 11:48:54 -0400 Subject: sentserver should be multi-threaded --- dpmert/Jamfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'dpmert') diff --git a/dpmert/Jamfile b/dpmert/Jamfile index 2bbf4f18..bc4b079b 100644 --- a/dpmert/Jamfile +++ b/dpmert/Jamfile @@ -23,7 +23,7 @@ lib dpmert : all_tests [ glob *_test.cc ] : dpmert : $(TOP)/dpmert/test_data ; -exe sentserver : sentserver.c ; +exe sentserver : sentserver.c : multi ; exe sentclient : sentclient.c ; exe mr_dpmert_generate_mapper_input : mr_dpmert_generate_mapper_input.cc dpmert ..//boost_program_options ; exe mr_dpmert_map : mr_dpmert_map.cc dpmert ..//boost_program_options ; -- cgit v1.2.3 From b9a3229520530cc382542456d2bd07d8707c5998 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Fri, 25 May 2012 11:51:46 -0400 Subject: Fix path to test_data --- dpmert/lo_test.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'dpmert') diff --git a/dpmert/lo_test.cc b/dpmert/lo_test.cc index f6df523e..2daf87bb 100644 --- a/dpmert/lo_test.cc +++ b/dpmert/lo_test.cc @@ -118,13 +118,15 @@ BOOST_AUTO_TEST_CASE( TestS1) { to_optimize.push_back(fPhraseModel_1); to_optimize.push_back(fPhraseModel_2); + std::string path(boost::unit_test::framework::master_test_suite().argc == 2 ? boost::unit_test::framework::master_test_suite().argv[1] : "test_data"); + Hypergraph hg; - ReadFile rf("./test_data/0.json.gz"); + ReadFile rf(path + "/0.json.gz"); HypergraphIO::ReadFromJSON(rf.stream(), &hg); hg.Reweight(wts); Hypergraph hg2; - ReadFile rf2("./test_data/1.json.gz"); + ReadFile rf2(path + "/1.json.gz"); HypergraphIO::ReadFromJSON(rf2.stream(), &hg2); hg2.Reweight(wts); -- cgit v1.2.3