more bjam stuff, more cleanup

23 files changed, 161 insertions, 73 deletions

diff --git a/Jamroot b/Jamroot
index 8be59e5a..edc28e23 100644
--- a/Jamroot
+++ b/Jamroot
@@ -30,9 +30,9 @@ if [ test_header boost/serialization/map.hpp ] && [ test_library boost_serializa
 project : requirements $(requirements) <toolset>darwin:<link>static ;
 project : default-build <threading>single <warnings>on <variant>release ;
 
-install-bin-libs utils//programs mteval//programs klm/lm//programs training//liblbfgs decoder//cdec creg//creg phrasinator//programs mira//kbest_mira ;
+install-bin-libs dpmert//programs utils//programs mteval//programs klm/lm//programs training//liblbfgs decoder//cdec creg//creg phrasinator//programs mira//kbest_mira ;
 
-build-projects mteval decoder klm/lm training/liblbfgs creg ;
+build-projects mteval decoder dpmert klm/lm training/liblbfgs creg ;
 
 #Compile everything ending with _test.cc into a test and run it.
 rule all_tests ( targets * : dependencies : properties * ) {
diff --git a/creg/README b/creg/README
index 28a71d11..208cabf5 100644
--- a/creg/README
+++ b/creg/README
@@ -15,3 +15,6 @@ Linear regression example (training and testing):
   $ ./creg -n -x test_data/auto-mpg.trainfeat -y test_data/auto-mpg.trainresp --l2 1000 \
         -t test_data/auto-mpg.testfeat -s test_data/auto-mpg.testresp > weights.txt
 
+Ordinal regression example
+  $ ./creg -o -x test_data/shuttle.trainfeat -y test_data/shuttle.trainresp
+
diff --git a/creg/test_data/shuttle.testfeat b/creg/test_data/shuttle.testfeat
new file mode 100644
index 00000000..31c07017
--- /dev/null
+++ b/creg/test_data/shuttle.testfeat
@@ -0,0 +1 @@
+Challenger	{"temp":31, "date":9524}
diff --git a/creg/test_data/shuttle.testresp b/creg/test_data/shuttle.testresp
new file mode 100644
index 00000000..249e84c7
--- /dev/null
+++ b/creg/test_data/shuttle.testresp
@@ -0,0 +1 @@
+Challenger	2
diff --git a/creg/test_data/shuttle.trainfeat b/creg/test_data/shuttle.trainfeat
new file mode 100644
index 00000000..c17d0f7f
--- /dev/null
+++ b/creg/test_data/shuttle.trainfeat
@@ -0,0 +1,23 @@
+1	{"temp":66, "date":7772}
+2	{"temp":70, "date":7986}
+3	{"temp":69, "date":8116}
+5	{"temp":68, "date":8350}
+6	{"temp":67, "date":8494}
+7	{"temp":72, "date":8569}
+8	{"temp":73, "date":8642}
+9	{"temp":70, "date":8732}
+10	{"temp":57, "date":8799}
+11	{"temp":63, "date":8862}
+12	{"temp":70, "date":9008}
+13	{"temp":78, "date":9044}
+14	{"temp":67, "date":9078}
+15	{"temp":53, "date":9155}
+16	{"temp":67, "date":9233}
+17	{"temp":75, "date":9250}
+18	{"temp":70, "date":9299}
+19	{"temp":81, "date":9341}
+20	{"temp":76, "date":9370}
+21	{"temp":79, "date":9407}
+22	{"temp":75, "date":9434}
+23	{"temp":76, "date":9461}
+24	{"temp":58, "date":9508}
diff --git a/creg/test_data/shuttle.trainresp b/creg/test_data/shuttle.trainresp
new file mode 100644
index 00000000..53e58f2c
--- /dev/null
+++ b/creg/test_data/shuttle.trainresp
@@ -0,0 +1,23 @@
+1	0
+2	1
+3	0
+5	0
+6	1
+7	0
+8	0
+9	0
+10	1
+11	2
+12	2
+13	0
+14	0
+15	2
+16	2
+17	2
+18	2
+19	1
+20	1
+21	0
+22	2
+23	1
+24	2
diff --git a/decoder/bottom_up_parser.cc b/decoder/bottom_up_parser.cc
index 63939221..1f262747 100644
--- a/decoder/bottom_up_parser.cc
+++ b/decoder/bottom_up_parser.cc
@@ -154,7 +154,7 @@ PassiveChart::PassiveChart(const string& goal,
     goal_idx_(-1),
     lc_fid_(FD::Convert("LatticeCost")) {
   act_chart_.resize(grammars_.size());
-  for (int i = 0; i < grammars_.size(); ++i)
+  for (unsigned i = 0; i < grammars_.size(); ++i)
     act_chart_[i] = new ActiveChart(forest, *this);
   if (!kGOAL) kGOAL = TD::Convert("Goal") * -1;
   if (!SILENT) cerr << "  Goal category: [" << goal << ']' << endl;
@@ -204,12 +204,12 @@ void PassiveChart::ApplyRules(const int i,
 
 void PassiveChart::ApplyUnaryRules(const int i, const int j) {
   const vector<int>& nodes = chart_(i,j);  // reference is important!
-  for (int gi = 0; gi < grammars_.size(); ++gi) {
+  for (unsigned gi = 0; gi < grammars_.size(); ++gi) {
     if (!grammars_[gi]->HasRuleForSpan(i,j,input_.Distance(i,j))) continue;
-    for (int di = 0; di < nodes.size(); ++di) {
+    for (unsigned di = 0; di < nodes.size(); ++di) {
       const WordID& cat = forest_->nodes_[nodes[di]].cat_;
       const vector<TRulePtr>& unaries = grammars_[gi]->GetUnaryRulesForRHS(cat);
-      for (int ri = 0; ri < unaries.size(); ++ri) {
+      for (unsigned ri = 0; ri < unaries.size(); ++ri) {
         // cerr << "At (" << i << "," << j << "): applying " << unaries[ri]->AsString() << endl;
         const Hypergraph::TailNodeVector ant(1, nodes[di]);
         ApplyRule(i, j, unaries[ri], ant, 0);  // may update nodes
@@ -224,15 +224,15 @@ bool PassiveChart::Parse() {
   size_t res = min(static_cast<size_t>(2000000), static_cast<size_t>(in_size_2 * 1000));
   forest_->edges_.reserve(res);
   goal_idx_ = -1;
-  for (int gi = 0; gi < grammars_.size(); ++gi)
+  for (unsigned gi = 0; gi < grammars_.size(); ++gi)
     act_chart_[gi]->SeedActiveChart(*grammars_[gi]);
 
   if (!SILENT) cerr << "    ";
-  for (int l=1; l<input_.size()+1; ++l) {
+  for (unsigned l=1; l<input_.size()+1; ++l) {
     if (!SILENT) cerr << '.';
-    for (int i=0; i<input_.size() + 1 - l; ++i) {
-      int j = i + l;
-      for (int gi = 0; gi < grammars_.size(); ++gi) {
+    for (unsigned i=0; i<input_.size() + 1 - l; ++i) {
+      unsigned j = i + l;
+      for (unsigned gi = 0; gi < grammars_.size(); ++gi) {
         const Grammar& g = *grammars_[gi];
         if (g.HasRuleForSpan(i, j, input_.Distance(i, j))) {
           act_chart_[gi]->AdvanceDotsForAllItemsInCell(i, j, input_);
@@ -248,7 +248,7 @@ bool PassiveChart::Parse() {
       }
       ApplyUnaryRules(i,j);
 
-      for (int gi = 0; gi < grammars_.size(); ++gi) {
+      for (unsigned gi = 0; gi < grammars_.size(); ++gi) {
         const Grammar& g = *grammars_[gi];
           // deal with non-terminals that were just proved
           if (g.HasRuleForSpan(i, j, input_.Distance(i,j)))
@@ -256,7 +256,7 @@ bool PassiveChart::Parse() {
       }
     }
     const vector<int>& dh = chart_(0, input_.size());
-    for (int di = 0; di < dh.size(); ++di) {
+    for (unsigned di = 0; di < dh.size(); ++di) {
       const Hypergraph::Node& node = forest_->nodes_[dh[di]];
       if (node.cat_ == goal_cat_) {
         Hypergraph::TailNodeVector ant(1, node.id_);
@@ -272,7 +272,7 @@ bool PassiveChart::Parse() {
 }
 
 PassiveChart::~PassiveChart() {
-  for (int i = 0; i < act_chart_.size(); ++i)
+  for (unsigned i = 0; i < act_chart_.size(); ++i)
     delete act_chart_[i];
 }
 
diff --git a/decoder/ff_dwarf.cc b/decoder/ff_dwarf.cc
index 3daa85ac..43528405 100644
--- a/decoder/ff_dwarf.cc
+++ b/decoder/ff_dwarf.cc
@@ -519,7 +519,7 @@ void Dwarf::neighboringFWs(const Lattice& l, const int& i, const int& j, const m
   while (idx>=0) {
     if (l[idx].size()>0) { 
       if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) {
-        *lfw++;  
+        lfw++;  
       }
     }
     idx-=l[idx][0].dist2next;
@@ -528,7 +528,7 @@ void Dwarf::neighboringFWs(const Lattice& l, const int& i, const int& j, const m
   while (idx<l.size()) {
     if (l[idx].size()>0) { 
       if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) {
-        *rfw++;
+        rfw++;
       }
     }
     idx+=l[idx][0].dist2next;
@@ -787,7 +787,7 @@ bool Dwarf::generalizeOrientation(CountTable* table, const std::map<WordID,WordI
       }
     }
   }
-
+  return false; // no idea if this is right
 }
  
 
diff --git a/decoder/hg.h b/decoder/hg.h
index dfa4ac6d..5f6d57ab 100644
--- a/decoder/hg.h
+++ b/decoder/hg.h
@@ -396,7 +396,7 @@ public:
   // (inner product) to set the edge probabilities
   template <class V>
   void Reweight(const V& weights) {
-    for (int i = 0; i < edges_.size(); ++i) {
+    for (unsigned i = 0; i < edges_.size(); ++i) {
       Edge& e = edges_[i];
       e.edge_prob_.logeq(e.feature_values_.dot(weights));
     }
diff --git a/decoder/hg_io.cc b/decoder/hg_io.cc
index 734c2ce8..3321558d 100644
--- a/decoder/hg_io.cc
+++ b/decoder/hg_io.cc
@@ -488,13 +488,13 @@ int getInt(const std::string& in, int &c)
 #define MAX_NODES 100000000
 // parse ('foo', 0.23)
 void ReadPLFEdge(const std::string& in, int &c, int cur_node, Hypergraph* hg) {
-  if (get(in,c++) != '(') { assert(!"PCN/PLF parse error: expected ( at start of cn alt block\n"); }
+  if (get(in,c++) != '(') { cerr << "PCN/PLF parse error: expected (\n"; abort(); }
   vector<WordID> ewords(2, 0);
   ewords[1] = TD::Convert(getEscapedString(in,c));
   TRulePtr r(new TRule(ewords));
   r->ComputeArity();
   // cerr << "RULE: " << r->AsString() << endl;
-  if (get(in,c++) != ',') { cerr << in << endl; assert(!"PCN/PLF parse error: expected , after string\n"); }
+  if (get(in,c++) != ',') { cerr << in << endl; cerr << "PCN/PLF parse error: expected , after string\n"; abort(); }
   size_t cnNext = 1;
   std::vector<float> probs;
   probs.push_back(getFloat(in,c));
@@ -508,10 +508,9 @@ void ReadPLFEdge(const std::string& in, int &c, int cur_node, Hypergraph* hg) {
   if (probs.size()>1) {
     cnNext = static_cast<size_t>(probs.back());
     probs.pop_back();
-    if (cnNext < 1) { cerr << cnNext << endl;
-             assert(!"PCN/PLF parse error: bad link length at last element of cn alt block\n"); }
+    if (cnNext < 1) { cerr << cnNext << endl << "PCN/PLF parse error: bad link length at last element of cn alt block\n"; abort(); }
   }
-  if (get(in,c++) != ')') { assert(!"PCN/PLF parse error: expected ) at end of cn alt block\n"); }
+  if (get(in,c++) != ')') { cerr << "PCN/PLF parse error: expected ) at end of cn alt block\n"; abort(); }
   eatws(in,c);
   Hypergraph::TailNodeVector tail(1, cur_node);
   Hypergraph::Edge* edge = hg->AddEdge(r, tail);
diff --git a/decoder/inside_outside.h b/decoder/inside_outside.h
index dc96f1a9..2ded328d 100644
--- a/decoder/inside_outside.h
+++ b/decoder/inside_outside.h
@@ -31,24 +31,24 @@ template<class WeightType, class WeightFunction>
 WeightType Inside(const Hypergraph& hg,
                   std::vector<WeightType>* result = NULL,
                   const WeightFunction& weight = WeightFunction()) {
-  const int num_nodes = hg.nodes_.size();
+  const unsigned num_nodes = hg.nodes_.size();
   std::vector<WeightType> dummy;
   std::vector<WeightType>& inside_score = result ? *result : dummy;
   inside_score.clear();
   inside_score.resize(num_nodes);
 //  std::fill(inside_score.begin(), inside_score.end(), WeightType()); // clear handles
-  for (int i = 0; i < num_nodes; ++i) {
+  for (unsigned i = 0; i < num_nodes; ++i) {
     WeightType* const cur_node_inside_score = &inside_score[i];
     Hypergraph::EdgesVector const& in=hg.nodes_[i].in_edges_;
-    const int num_in_edges = in.size();
+    const unsigned num_in_edges = in.size();
     if (num_in_edges == 0) {
       *cur_node_inside_score = WeightType(1); //FIXME: why not call weight(edge) instead?
       continue;
     }
-    for (int j = 0; j < num_in_edges; ++j) {
+    for (unsigned j = 0; j < num_in_edges; ++j) {
       const Hypergraph::Edge& edge = hg.edges_[in[j]];
       WeightType score = weight(edge);
-      for (int k = 0; k < edge.tail_nodes_.size(); ++k) {
+      for (unsigned k = 0; k < edge.tail_nodes_.size(); ++k) {
         const int tail_node_index = edge.tail_nodes_[k];
         score *= inside_score[tail_node_index];
       }
diff --git a/decoder/kbest.h b/decoder/kbest.h
index 03a8311c..9af3a20e 100644
--- a/decoder/kbest.h
+++ b/decoder/kbest.h
@@ -43,7 +43,7 @@ namespace KBest {
       traverse(tf), w(wf), g(hg), nds(g.nodes_.size()), k_prime(k) {}
 
     ~KBestDerivations() {
-      for (int i = 0; i < freelist.size(); ++i)
+      for (unsigned i = 0; i < freelist.size(); ++i)
         delete freelist[i];
     }
 
@@ -86,7 +86,7 @@ namespace KBest {
 //      Hypergraph::Edge const * operator ->() const { return d->edge; }
     };
 
-    EdgeHandle operator()(int t,int taili,EdgeHandle const& parent) const {
+    EdgeHandle operator()(unsigned t,unsigned taili,EdgeHandle const& parent) const {
       return EdgeHandle(nds[t].D[parent.d->j[taili]]);
     }
 
@@ -98,7 +98,7 @@ namespace KBest {
       size_t operator()(const Derivation* d) const {
         size_t x = 5381;
         x = ((x << 5) + x) ^ d->edge->id_;
-        for (int i = 0; i < d->j.size(); ++i)
+        for (unsigned i = 0; i < d->j.size(); ++i)
           x = ((x << 5) + x) ^ d->j[i];
         return x;
       }
@@ -121,7 +121,7 @@ namespace KBest {
       explicit NodeDerivationState(const DerivationFilter& f = DerivationFilter()) : filter(f) {}
     };
 
-    Derivation* LazyKthBest(int v, int k) {
+    Derivation* LazyKthBest(unsigned v, unsigned k) {
       NodeDerivationState& s = GetCandidates(v);
       CandidateHeap& cand = s.cand;
       DerivationList& D = s.D;
@@ -139,7 +139,7 @@ namespace KBest {
           Derivation* d = cand.back();
           cand.pop_back();
           std::vector<const T*> ants(d->edge->Arity());
-          for (int j = 0; j < ants.size(); ++j)
+          for (unsigned j = 0; j < ants.size(); ++j)
             ants[j] = &LazyKthBest(d->edge->tail_nodes_[j], d->j[j])->yield;
           traverse(*d->edge, ants, &d->yield);
           if (!filter(d->yield)) {
@@ -171,12 +171,12 @@ namespace KBest {
       return freelist.back();
     }
 
-    NodeDerivationState& GetCandidates(int v) {
+    NodeDerivationState& GetCandidates(unsigned v) {
       NodeDerivationState& s = nds[v];
       if (!s.D.empty() || !s.cand.empty()) return s;
 
       const Hypergraph::Node& node = g.nodes_[v];
-      for (int i = 0; i < node.in_edges_.size(); ++i) {
+      for (unsigned i = 0; i < node.in_edges_.size(); ++i) {
         const Hypergraph::Edge& edge = g.edges_[node.in_edges_[i]];
         SmallVectorInt jv(edge.Arity(), 0);
         Derivation* d = CreateDerivation(edge, jv);
@@ -184,7 +184,7 @@ namespace KBest {
         s.cand.push_back(d);
       }
 
-      const int effective_k = std::min(k_prime, s.cand.size());
+      const unsigned effective_k = std::min(k_prime, s.cand.size());
       const typename CandidateHeap::iterator kth = s.cand.begin() + effective_k;
       std::nth_element(s.cand.begin(), kth, s.cand.end(), DerivationCompare());
       s.cand.resize(effective_k);
@@ -194,7 +194,7 @@ namespace KBest {
     }
 
     void LazyNext(const Derivation* d, CandidateHeap* cand, UniqueDerivationSet* ds) {
-      for (int i = 0; i < d->j.size(); ++i) {
+      for (unsigned i = 0; i < d->j.size(); ++i) {
         SmallVectorInt j = d->j;
         ++j[i];
         const Derivation* ant = LazyKthBest(d->edge->tail_nodes_[i], j[i]);
@@ -205,8 +205,12 @@ namespace KBest {
             if (new_d) {
               cand->push_back(new_d);
               std::push_heap(cand->begin(), cand->end(), HeapCompare());
+#ifdef NDEBUG
+              ds->insert(new_d).second;  // insert into uniqueness set
+#else
               bool inserted = ds->insert(new_d).second;  // insert into uniqueness set
               assert(inserted);
+#endif
             }
           }
         }
diff --git a/decoder/trule.cc b/decoder/trule.cc
index 141b8faa..5ebc4c16 100644
--- a/decoder/trule.cc
+++ b/decoder/trule.cc
@@ -18,7 +18,7 @@ bool TRule::IsGoal() const {
 }
 
 static WordID ConvertTrgString(const string& w) {
-  int len = w.size();
+  const unsigned len = w.size();
   WordID id = 0;
   // [X,0] or [0]
   // for target rules, we ignore the category, just keep the index
@@ -33,7 +33,7 @@ static WordID ConvertTrgString(const string& w) {
 }
 
 static WordID ConvertSrcString(const string& w, bool mono = false) {
-  int len = w.size();
+  const unsigned len = w.size();
   // [X,0]
   // for source rules, we keep the category and ignore the index (source rules are
   // always numbered 1, 2, 3...
@@ -60,7 +60,7 @@ static WordID ConvertSrcString(const string& w, bool mono = false) {
 
 static WordID ConvertLHS(const string& w) {
   if (w[0] == '[') {
-    int len = w.size();
+    const unsigned len = w.size();
     if (len < 3) { cerr << "Format error: " << w << endl; exit(1); }
     return TD::Convert(w.substr(1, len-2)) * -1;
   } else {
@@ -143,15 +143,15 @@ bool TRule::ReadFromString(const string& line, bool strict, bool mono) {
       string ss;
       getline(is, ss);
       //cerr << "L: " << ss << endl;
-      int start = 0;
-      int len = ss.size();
+      unsigned start = 0;
+      unsigned len = ss.size();
       const size_t ppos = ss.find(" |||");
       if (ppos != string::npos) { len = ppos; }
       while (start < len) {
         while(start < len && (ss[start] == ' ' || ss[start] == ';'))
           ++start;
         if (start == len) break;
-        int end = start + 1;
+        unsigned end = start + 1;
         while(end < len && (ss[end] != '=' && ss[end] != ' ' && ss[end] != ';'))
           ++end;
         if (end == len || ss[end] == ' ' || ss[end] == ';') {
@@ -188,7 +188,7 @@ bool TRule::ReadFromString(const string& line, bool strict, bool mono) {
     while(is>>w && w!="|||") { e_.push_back(ConvertTrgString(w)); }
     f_ = e_;
     int x = ConvertLHS("[X]");
-    for (int i = 0; i < f_.size(); ++i)
+    for (unsigned i = 0; i < f_.size(); ++i)
       if (f_[i] <= 0) { f_[i] = x; }
   } else {
     cerr << "F: " << format << endl;
@@ -197,7 +197,7 @@ bool TRule::ReadFromString(const string& line, bool strict, bool mono) {
   if (mono) {
     e_ = f_;
     int ci = 0;
-    for (int i = 0; i < e_.size(); ++i)
+    for (unsigned i = 0; i < e_.size(); ++i)
       if (e_[i] < 0)
         e_[i] = ci--;
   }
@@ -208,7 +208,7 @@ bool TRule::ReadFromString(const string& line, bool strict, bool mono) {
 bool TRule::SanityCheck() const {
   vector<int> used(f_.size(), 0);
   int ac = 0;
-  for (int i = 0; i < e_.size(); ++i) {
+  for (unsigned i = 0; i < e_.size(); ++i) {
     int ind = e_[i];
     if (ind > 0) continue;
     ind = -ind;
@@ -238,7 +238,7 @@ string TRule::AsString(bool verbose) const {
   if (lhs_ && verbose) {
     os << '[' << TD::Convert(lhs_ * -1) << "] |||";
   }
-  for (int i = 0; i < f_.size(); ++i) {
+  for (unsigned i = 0; i < f_.size(); ++i) {
     const WordID& w = f_[i];
     if (w < 0) {
       int wi = w * -1;
@@ -249,7 +249,7 @@ string TRule::AsString(bool verbose) const {
     }
   }
   os << " ||| ";
-  for (int i =0; i<e_.size(); ++i) {
+  for (unsigned i =0; i<e_.size(); ++i) {
     if (i) os << ' ';
     const WordID& w = e_[i];
     if (w < 1)
@@ -261,7 +261,7 @@ string TRule::AsString(bool verbose) const {
     os << " ||| " << scores_;
     if (!a_.empty()) {
       os << " |||";
-      for (int i = 0; i < a_.size(); ++i)
+      for (unsigned i = 0; i < a_.size(); ++i)
         os << ' ' << a_[i];
     }
   }
diff --git a/dpmert/Jamfile b/dpmert/Jamfile
new file mode 100644
index 00000000..d96c592c
--- /dev/null
+++ b/dpmert/Jamfile
@@ -0,0 +1,30 @@
+import testing ;
+import lex ;
+import option ;
+
+lib dpmert : 
+  ces.cc
+  error_surface.cc
+  line_optimizer.cc
+  mert_geometry.cc
+  ..//utils
+  ..//mteval
+  ..//decoder
+  ../klm/lm//kenlm
+  ..//boost_program_options
+  : <include>.
+  : :
+  <library>..//utils
+  <library>..//mteval
+  <library>../klm/lm//kenlm
+  <library>..//boost_program_options
+  <include>.
+  ;
+
+all_tests [ glob *_test.cc ] : dpmert : <testing.arg>$(TOP)/dpmert/test_data ;
+
+exe sentserver : sentserver.c ;
+exe sentclient : sentclient.c ;
+exe mr_dpmert_generate_mapper_input : mr_dpmert_generate_mapper_input.cc dpmert ..//boost_program_options ;
+exe mr_dpmert_map : mr_dpmert_map.cc dpmert ..//boost_program_options ;
+exe mr_dpmert_reduce : mr_dpmert_reduce.cc dpmert ..//boost_program_options ;
diff --git a/dpmert/ces.cc b/dpmert/ces.cc
index c6cb1cdf..157b2d17 100644
--- a/dpmert/ces.cc
+++ b/dpmert/ces.cc
@@ -25,7 +25,7 @@ void ComputeErrorSurface(const SegmentEvaluator& ss,
   env->resize(ienv.size());
   SufficientStats prev_score; // defaults to 0
   int j = 0;
-  for (int i = 0; i < ienv.size(); ++i) {
+  for (unsigned i = 0; i < ienv.size(); ++i) {
     const MERTPoint& seg = *ienv[i];
     vector<WordID> trans;
 #if 0
diff --git a/dpmert/lo_test.cc b/dpmert/lo_test.cc
index d47a95b5..f6df523e 100644
--- a/dpmert/lo_test.cc
+++ b/dpmert/lo_test.cc
@@ -88,7 +88,7 @@ BOOST_AUTO_TEST_CASE(TestConvexHullInside) {
     if (!d) break;
     cerr << log(d->score) << " ||| " << TD::GetString(d->yield) << " ||| " << d->feature_values << endl;
   }
-  for (int i = 0; i < segs.size(); ++i) {
+  for (unsigned i = 0; i < segs.size(); ++i) {
     cerr << "seg=" << i << endl;
     vector<WordID> trans;
     segs[i]->ConstructTranslation(&trans);
@@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE( TestS1) {
      &rng,
      &axes);
   assert(axes.size() == 10 + to_optimize.size());
-  for (int i = 0; i < axes.size(); ++i)
+  for (unsigned i = 0; i < axes.size(); ++i)
     cerr << axes[i] << endl;
   const SparseVector<double>& axis = axes[0];
 
diff --git a/dpmert/mr_dpmert_generate_mapper_input.cc b/dpmert/mr_dpmert_generate_mapper_input.cc
index 59d4f24f..199cd23a 100644
--- a/dpmert/mr_dpmert_generate_mapper_input.cc
+++ b/dpmert/mr_dpmert_generate_mapper_input.cc
@@ -52,12 +52,15 @@ int main(int argc, char** argv) {
   Weights::InitFromFile(conf["weights"].as<string>(), &w, &features);
   Weights::InitSparseVector(w, &origin);
   const string forest_repository = conf["forest_repository"].as<string>();
-  assert(DirectoryExists(forest_repository));
+  if (!DirectoryExists(forest_repository)) {
+    cerr << "Forest repository directory " << forest_repository << " not found!\n";
+    return 1;
+  }
   if (conf.count("optimize_feature") > 0)
     features=conf["optimize_feature"].as<vector<string> >();
   vector<SparseVector<weight_t> > directions;
   vector<int> fids(features.size());
-  for (int i = 0; i < features.size(); ++i)
+  for (unsigned i = 0; i < features.size(); ++i)
     fids[i] = FD::Convert(features[i]);
   LineOptimizer::CreateOptimizationDirections(
      fids,
diff --git a/dpmert/mr_dpmert_map.cc b/dpmert/mr_dpmert_map.cc
index f3304f0f..d1efcf96 100644
--- a/dpmert/mr_dpmert_map.cc
+++ b/dpmert/mr_dpmert_map.cc
@@ -52,7 +52,7 @@ bool ReadSparseVectorString(const string& s, SparseVector<double>* v) {
   vector<string> fields;
   Tokenize(s, ';', &fields);
   if (fields.empty()) return false;
-  for (int i = 0; i < fields.size(); ++i) {
+  for (unsigned i = 0; i < fields.size(); ++i) {
     vector<string> pair(2);
     Tokenize(fields[i], '=', &pair);
     if (pair.size() != 2) {
diff --git a/pro-train/Makefile.am b/pro-train/Makefile.am
index 11d26211..a98dd245 100644
--- a/pro-train/Makefile.am
+++ b/pro-train/Makefile.am
@@ -8,6 +8,6 @@ mr_pro_map_SOURCES = mr_pro_map.cc
 mr_pro_map_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lz
 
 mr_pro_reduce_SOURCES = mr_pro_reduce.cc
-mr_pro_reduce_LDADD = $(top_srcdir)/decoder/libcdec.a $(top_srcdir)/training/optimize.o $(top_srcdir)/training/liblbfgs/liblbfgs.a $(top_srcdir)/mteval/libmteval.a $(top_srcdir)/utils/libutils.a -lz
+mr_pro_reduce_LDADD = $(top_srcdir)/training/liblbfgs/liblbfgs.a $(top_srcdir)/utils/libutils.a -lz
 
 AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(GTEST_CPPFLAGS) -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I$(top_srcdir)/training
diff --git a/pro-train/mr_pro_reduce.cc b/pro-train/mr_pro_reduce.cc
index d3fb8026..9698bb5d 100644
--- a/pro-train/mr_pro_reduce.cc
+++ b/pro-train/mr_pro_reduce.cc
@@ -25,6 +25,7 @@ void InitCommandLine(int argc, char** argv, po::variables_map* conf) {
   opts.add_options()
         ("weights,w", po::value<string>(), "Weights from previous iteration (used as initialization and interpolation")
         ("regularization_strength,C",po::value<double>()->default_value(500.0), "l2 regularization strength")
+        ("l1",po::value<double>()->default_value(0.0), "l1 regularization strength")
         ("regularize_to_weights,y",po::value<double>()->default_value(5000.0), "Differences in learned weights to previous weights are penalized with an l2 penalty with this strength; 0.0 = no effect")
         ("memory_buffers,m",po::value<unsigned>()->default_value(100), "Number of memory buffers (LBFGS)")
         ("min_reg,r",po::value<double>()->default_value(0.01), "When tuning (-T) regularization strength, minimum regularization strenght")
@@ -180,12 +181,13 @@ struct ProLoss {
 double LearnParameters(const vector<pair<bool, SparseVector<weight_t> > >& training,
                        const vector<pair<bool, SparseVector<weight_t> > >& testing,
                        const double C,
+                       const double C1,
                        const double T,
                        const unsigned memory_buffers,
                        const vector<weight_t>& prev_x,
                        vector<weight_t>* px) {
   ProLoss loss(training, testing, C, T, prev_x);
-  LBFGS<ProLoss> lbfgs(px, loss, 0.0, memory_buffers);
+  LBFGS<ProLoss> lbfgs(px, loss, C1, memory_buffers);
   lbfgs.MinimizeFunction();
   return loss.tppl;
 }
@@ -203,6 +205,7 @@ int main(int argc, char** argv) {
   const double min_reg = conf["min_reg"].as<double>();
   const double max_reg = conf["max_reg"].as<double>();
   double C = conf["regularization_strength"].as<double>(); // will be overridden if parameter is tuned
+  double C1 = conf["l1"].as<double>(); // will be overridden if parameter is tuned
   const double T = conf["regularize_to_weights"].as<double>();
   assert(C >= 0.0);
   assert(min_reg >= 0.0);
@@ -239,7 +242,7 @@ int main(int argc, char** argv) {
     cerr << "SWEEP FACTOR: " << sweep_factor << endl;
     while(C < max_reg) {
       cerr << "C=" << C << "\tT=" <<T << endl;
-      tppl = LearnParameters(training, testing, C, T, conf["memory_buffers"].as<unsigned>(), prev_x, &x);
+      tppl = LearnParameters(training, testing, C, C1, T, conf["memory_buffers"].as<unsigned>(), prev_x, &x);
       sp.push_back(make_pair(C, tppl));
       C *= sweep_factor;
     }
@@ -262,7 +265,7 @@ int main(int argc, char** argv) {
     }
     C = sp[best_i].first;
   }  // tune regularizer
-  tppl = LearnParameters(training, testing, C, T, conf["memory_buffers"].as<unsigned>(), prev_x, &x);
+  tppl = LearnParameters(training, testing, C, C1, T, conf["memory_buffers"].as<unsigned>(), prev_x, &x);
   if (conf.count("weights")) {
     for (int i = 1; i < x.size(); ++i) {
       x[i] = (x[i] * psi) + prev_x[i] * (1.0 - psi);
diff --git a/utils/Jamfile b/utils/Jamfile
index 53a51277..4444b25f 100644
--- a/utils/Jamfile
+++ b/utils/Jamfile
@@ -24,11 +24,9 @@ lib utils :
   ..//z
   : <include>.. <include>. : : <include>.. <include>. ;
 
-exe ts : ts.cc utils ;
 exe atools : atools.cc utils ..//boost_program_options ;
-exe phmt : phmt.cc utils ;
 exe reconstruct_weights : reconstruct_weights.cc utils ..//boost_program_options ;
 
 alias programs : reconstruct_weights atools ;
 
-all_tests : utils ;
+all_tests [ glob *_test.cc phmt.cc ts.cc ] : utils : <testing.arg>$(TOP)/utils/test_data ;
diff --git a/utils/null_traits.h b/utils/null_traits.h
index fac857d9..7b2d32d0 100644
--- a/utils/null_traits.h
+++ b/utils/null_traits.h
@@ -3,23 +3,23 @@
 
 template <class V>
 struct null_traits {
-  static V null; //TODO: maybe take out default null and make ppl explicitly define?  they may be surprised that they need to when they include a header lib that uses null_traits
+  static V xnull; //TODO: maybe take out default null and make ppl explicitly define?  they may be surprised that they need to when they include a header lib that uses null_traits
 };
 // global bool is_null(V const& v)
 
 // definitely override this, and possibly set_null and is_null.  that's the point.
 template <class V>
-V null_traits<V>::null;
+V null_traits<V>::xnull;
 //TODO: are we getting single init of the static null object?
 
 template <class V>
 void set_null(V &v) {
-  v=null_traits<V>::null;
+  v=null_traits<V>::xnull;
 }
 
 template <class V>
 void is_null(V const& v) {
-  return v==null_traits<V>::null;
+  return v==null_traits<V>::xnull;
 }
 
 
diff --git a/utils/stringlib.h b/utils/stringlib.h
index f457e1e4..13d14dbf 100644
--- a/utils/stringlib.h
+++ b/utils/stringlib.h
@@ -97,9 +97,9 @@ inline std::string Trim(const std::string& str, const std::string& dropChars = "
 
 inline void Tokenize(const std::string& str, char delimiter, std::vector<std::string>* res) {
   std::string s = str;
-  int last = 0;
+  unsigned last = 0;
   res->clear();
-  for (int i=0; i < s.size(); ++i)
+  for (unsigned i=0; i < s.size(); ++i)
     if (s[i] == delimiter) {
       s[i]=0;
       if (last != i) {
@@ -120,14 +120,14 @@ inline unsigned NTokens(const std::string& str, char delimiter)
 
 inline std::string LowercaseString(const std::string& in) {
   std::string res(in.size(),' ');
-  for (int i = 0; i < in.size(); ++i)
+  for (unsigned i = 0; i < in.size(); ++i)
     res[i] = tolower(in[i]);
   return res;
 }
 
 inline std::string UppercaseString(const std::string& in) {
   std::string res(in.size(),' ');
-  for (int i = 0; i < in.size(); ++i)
+  for (unsigned i = 0; i < in.size(); ++i)
     res[i] = toupper(in[i]);
   return res;
 }
@@ -146,8 +146,8 @@ inline int CountSubstrings(const std::string& str, const std::string& sub) {
 
 inline int SplitOnWhitespace(const std::string& in, std::vector<std::string>* out) {
   out->clear();
-  int i = 0;
-  int start = 0;
+  unsigned i = 0;
+  unsigned start = 0;
   std::string cur;
   while(i < in.size()) {
     if (in[i] == ' ' || in[i] == '\t') {
@@ -249,7 +249,7 @@ inline void SplitCommandAndParam(const std::string& in, std::string* cmd, std::s
   SplitOnWhitespace(in, &x);
   if (x.size() == 0) return;
   *cmd = x[0];
-  for (int i = 1; i < x.size(); ++i) {
+  for (unsigned i = 1; i < x.size(); ++i) {
     if (i > 1) { *param += " "; }
     *param += x[i];
   }