Merge remote-tracking branch 'upstream/master'

4 files changed, 69 insertions, 4 deletions

diff --git a/training/Makefile.am b/training/Makefile.am
index 68ebfab4..4cef0d5b 100644
--- a/training/Makefile.am
+++ b/training/Makefile.am
@@ -26,6 +26,7 @@ TESTS = lbfgs_test optimize_test
 noinst_LIBRARIES = libtraining.a
 libtraining_a_SOURCES = \
   candidate_set.cc \
+  entropy.cc \
   optimize.cc \
   online_optimizer.cc \
   risk.cc
diff --git a/training/candidate_set.cc b/training/candidate_set.cc
index 8c086ece..087efec3 100644
--- a/training/candidate_set.cc
+++ b/training/candidate_set.cc
@@ -4,6 +4,7 @@
 
 #include <boost/functional/hash.hpp>
 
+#include "verbose.h"
 #include "ns.h"
 #include "filelib.h"
 #include "wordid.h"
@@ -118,7 +119,7 @@ void CandidateSet::WriteToFile(const string& file) const {
 }
 
 void CandidateSet::ReadFromFile(const string& file) {
-  cerr << "Reading candidates from " << file << endl;
+  if(!SILENT) cerr << "Reading candidates from " << file << endl;
   ReadFile rf(file);
   istream& in = *rf.stream();
   string cand;
@@ -133,11 +134,11 @@ void CandidateSet::ReadFromFile(const string& file) {
     ParseSparseVector(feats, 0, &cs.back().fmap);
     cs.back().eval_feats = SufficientStats(ss);
   }
-  cerr << "  read " << cs.size() << " candidates\n";
+  if(!SILENT) cerr << "  read " << cs.size() << " candidates\n";
 }
 
 void CandidateSet::Dedup() {
-  cerr << "Dedup in=" << cs.size();
+  if(!SILENT) cerr << "Dedup in=" << cs.size();
   tr1::unordered_set<Candidate, CandidateHasher, CandidateCompare> u;
   while(cs.size() > 0) {
     u.insert(cs.back());
@@ -148,7 +149,7 @@ void CandidateSet::Dedup() {
     cs.push_back(*it);
     it = u.erase(it);
   }
-  cerr << "  out=" << cs.size() << endl;
+  if(!SILENT) cerr << "  out=" << cs.size() << endl;
 }
 
 void CandidateSet::AddKBestCandidates(const Hypergraph& hg, size_t kbest_size, const SegmentEvaluator* scorer) {
diff --git a/training/entropy.cc b/training/entropy.cc
new file mode 100644
index 00000000..4fdbe2be
--- /dev/null
+++ b/training/entropy.cc
@@ -0,0 +1,41 @@
+#include "entropy.h"
+
+#include "prob.h"
+#include "candidate_set.h"
+
+using namespace std;
+
+namespace training {
+
+// see Mann and McCallum "Efficient Computation of Entropy Gradient ..." for
+// a mostly clear derivation of:
+//   g = E[ F(x,y) * log p(y|x) ] + H(y | x) * E[ F(x,y) ]
+double CandidateSetEntropy::operator()(const vector<double>& params,
+                                       SparseVector<double>* g) const {
+  prob_t z;
+  vector<double> dps(cands_.size());
+  for (unsigned i = 0; i < cands_.size(); ++i) {
+    dps[i] = cands_[i].fmap.dot(params);
+    const prob_t u(dps[i], init_lnx());
+    z += u;
+  }
+  const double log_z = log(z);
+
+  SparseVector<double> exp_feats;
+  double entropy = 0;
+  for (unsigned i = 0; i < cands_.size(); ++i) {
+    const double log_prob = cands_[i].fmap.dot(params) - log_z;
+    const double prob = exp(log_prob);
+    const double e_logprob = prob * log_prob;
+    entropy -= e_logprob;
+    if (g) {
+      (*g) += cands_[i].fmap * e_logprob;
+      exp_feats += cands_[i].fmap * prob;
+    }
+  }
+  if (g) (*g) += exp_feats * entropy;
+  return entropy;
+}
+
+}
+
diff --git a/training/entropy.h b/training/entropy.h
new file mode 100644
index 00000000..796589ca
--- /dev/null
+++ b/training/entropy.h
@@ -0,0 +1,22 @@
+#ifndef _CSENTROPY_H_
+#define _CSENTROPY_H_
+
+#include <vector>
+#include "sparse_vector.h"
+
+namespace training {
+  class CandidateSet;
+
+  class CandidateSetEntropy {
+   public:
+    explicit CandidateSetEntropy(const CandidateSet& cs) : cands_(cs) {}
+    // compute the entropy (expected log likelihood) of a CandidateSet
+    // (optional) the gradient of the entropy with respect to params
+    double operator()(const std::vector<double>& params,
+                      SparseVector<double>* g = NULL) const;
+   private:
+    const CandidateSet& cands_;
+  };
+};
+
+#endif