[python] MT evaluation

12 files changed, 4266 insertions, 322 deletions

diff --git a/python/cdec/__init__.py b/python/cdec/__init__.py
index eda81194..0d7b8782 100644
--- a/python/cdec/__init__.py
+++ b/python/cdec/__init__.py
@@ -1 +1,2 @@
 from _cdec import Decoder, Lattice
+import score
diff --git a/python/cdec/score.py b/python/cdec/score.py
new file mode 100644
index 00000000..c107446f
--- /dev/null
+++ b/python/cdec/score.py
@@ -0,0 +1 @@
+from _cdec import BLEU, TER
diff --git a/python/setup.py b/python/setup.py
index 49a1ec8f..cc8b289d 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -3,8 +3,8 @@ from distutils.extension import Extension
 from Cython.Distutils import build_ext
 import os
 
-INC = ['..', 'src/', '../decoder', '../utils']
-LIB = ['../decoder', '../utils', '../mteval', '../klm/lm', '../klm/util']
+INC = ['..', 'src/', '../decoder', '../utils', '../mteval']
+LIB = ['../decoder', '../utils', '../mteval', '../training', '../klm/lm', '../klm/util']
 
 BOOST_ROOT = os.getenv('BOOST_ROOT')
 if BOOST_ROOT:
@@ -18,7 +18,8 @@ ext_modules = [
         include_dirs=INC,
         library_dirs=LIB,
         libraries=['boost_program_options-mt', 'z',
-                   'cdec', 'utils', 'mteval', 'klm', 'klm_util'])
+                   'cdec', 'utils', 'mteval', 'training', 'klm', 'klm_util'],
+        extra_compile_args=['-DHAVE_CONFIG_H']),
 ]
 
 setup(
diff --git a/python/src/_cdec.cpp b/python/src/_cdec.cpp
index 0b9b9911..7d45e27f 100644
--- a/python/src/_cdec.cpp
+++ b/python/src/_cdec.cpp
@@ -1,4 +1,4 @@
-/* Generated by Cython 0.16 on Sat Jun 23 11:57:21 2012 */
+/* Generated by Cython 0.16 on Sun Jul  1 00:36:19 2012 */
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
@@ -279,11 +279,12 @@
 #include "utils/wordid.h"
 #include "utils/small_vector.h"
 #include "utils/sparse_vector.h"
-#include "utils/tdict.cc"
+#include "utils/tdict.h"
 #include "utils/verbose.h"
 #include "utils/fdict.h"
 #include "utils/filelib.h"
 #include "utils/sampler.h"
+#include <boost/shared_ptr.hpp>
 #include "decoder/lattice.h"
 #include "decoder/hg.h"
 #include "decoder/viterbi.h"
@@ -295,6 +296,8 @@
 #include "decoder/decoder.h"
 #include "observer.h"
 #include "decoder/kbest.h"
+#include "mteval/ns.h"
+#include "training/candidate_set.h"
 #ifdef _OPENMP
 #include <omp.h>
 #endif /* _OPENMP */
@@ -381,37 +384,66 @@ static const char *__pyx_f[] = {
   "vectors.pxi",
   "hypergraph.pxi",
   "lattice.pxi",
+  "mteval.pxi",
 };
 
 /*--- Type declarations ---*/
-struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample;
+struct __pyx_obj_5_cdec_Scorer;
+struct __pyx_obj_5_cdec_SufficientStats;
 struct __pyx_obj_5_cdec___pyx_scope_struct_1___iter__;
+struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features;
 struct __pyx_obj_5_cdec___pyx_scope_struct_3_kbest_tree;
 struct __pyx_obj_5_cdec_Decoder;
+struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set;
+struct __pyx_obj_5_cdec_SegmentEvaluator;
 struct __pyx_obj_5_cdec_SparseVector;
 struct __pyx_obj_5_cdec_Hypergraph;
 struct __pyx_obj_5_cdec_Lattice;
-struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__;
 struct __pyx_obj_5_cdec___pyx_scope_struct____iter__;
 struct __pyx_obj_5_cdec___pyx_scope_struct_2_kbest;
+struct __pyx_obj_5_cdec_Candidate;
+struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample;
 struct __pyx_obj_5_cdec_DenseVector;
+struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__;
+struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__;
+struct __pyx_opt_args_5_cdec_as_str;
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":56
- *         del derivations
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":3
+ * cimport mteval
  * 
- *     def sample(self, unsigned n):             # <<<<<<<<<<<<<<
- *         cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]()
- *         if self.rng == NULL:
+ * cdef char* as_str(sentence, error_msg='Cannot convert type %s to str'):             # <<<<<<<<<<<<<<
+ *     cdef bytes ret
+ *     if isinstance(sentence, unicode):
  */
-struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample {
+struct __pyx_opt_args_5_cdec_as_str {
+  int __pyx_n;
+  PyObject *error_msg;
+};
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":87
+ *         del cs
+ * 
+ * cdef class Scorer:             # <<<<<<<<<<<<<<
+ *     cdef string* name
+ * 
+ */
+struct __pyx_obj_5_cdec_Scorer {
   PyObject_HEAD
-  std::vector<HypergraphSampler::Hypothesis> *__pyx_v_hypos;
-  unsigned int __pyx_v_k;
-  unsigned int __pyx_v_n;
-  struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self;
-  PyObject *__pyx_v_sentence;
-  size_t __pyx_t_0;
-  unsigned int __pyx_t_1;
+  std::string *name;
+};
+
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":27
+ *             return fmap
+ * 
+ * cdef class SufficientStats:             # <<<<<<<<<<<<<<
+ *     cdef mteval.SufficientStats* stats
+ *     cdef mteval.EvaluationMetric* metric
+ */
+struct __pyx_obj_5_cdec_SufficientStats {
+  PyObject_HEAD
+  SufficientStats *stats;
+  EvaluationMetric *metric;
 };
 
 
@@ -433,6 +465,26 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_1___iter__ {
 };
 
 
+/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":56
+ *         del derivations
+ * 
+ *     def kbest_features(self, size):             # <<<<<<<<<<<<<<
+ *         cdef kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal]* derivations = new kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal](self.hg[0], size)
+ *         cdef kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal].Derivation* derivation
+ */
+struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features {
+  PyObject_HEAD
+  KBest::KBestDerivations<FastSparseVector<weight_t>,FeatureVectorTraversal>::Derivation *__pyx_v_derivation;
+  KBest::KBestDerivations<FastSparseVector<weight_t>,FeatureVectorTraversal> *__pyx_v_derivations;
+  struct __pyx_obj_5_cdec_SparseVector *__pyx_v_fmap;
+  unsigned int __pyx_v_k;
+  struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self;
+  PyObject *__pyx_v_size;
+  unsigned int __pyx_t_0;
+  long __pyx_t_1;
+};
+
+
 /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":44
  *         del derivations
  * 
@@ -454,7 +506,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_3_kbest_tree {
 
 
 /* "_cdec.pyx":15
- *     pass
+ * class ParseFailed(Exception): pass
  * 
  * cdef class Decoder:             # <<<<<<<<<<<<<<
  *     cdef decoder.Decoder* dec
@@ -467,6 +519,40 @@ struct __pyx_obj_5_cdec_Decoder {
 };
 
 
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":75
+ *         return sf
+ * 
+ *     def candidate_set(self, Hypergraph hypergraph, unsigned k):             # <<<<<<<<<<<<<<
+ *         cdef mteval.CandidateSet* cs = new mteval.CandidateSet()
+ *         cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get())
+ */
+struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set {
+  PyObject_HEAD
+  struct __pyx_obj_5_cdec_Candidate *__pyx_v_candidate;
+  training::CandidateSet *__pyx_v_cs;
+  struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_hypergraph;
+  unsigned int __pyx_v_i;
+  unsigned int __pyx_v_k;
+  struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self;
+  unsigned int __pyx_t_0;
+  unsigned int __pyx_t_1;
+};
+
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":59
+ *         return result
+ * 
+ * cdef class SegmentEvaluator:             # <<<<<<<<<<<<<<
+ *     cdef shared_ptr[mteval.SegmentEvaluator]* scorer
+ *     cdef mteval.EvaluationMetric* metric
+ */
+struct __pyx_obj_5_cdec_SegmentEvaluator {
+  PyObject_HEAD
+  boost::shared_ptr<SegmentEvaluator> *scorer;
+  EvaluationMetric *metric;
+};
+
+
 /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":32
  *         return sparse
  * 
@@ -507,22 +593,6 @@ struct __pyx_obj_5_cdec_Lattice {
 };
 
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":50
- *         return hypergraph.AsPLF(self.lattice[0], True).c_str()
- * 
- *     def __iter__(self):             # <<<<<<<<<<<<<<
- *         cdef unsigned i
- *         for i in range(len(self)):
- */
-struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ {
-  PyObject_HEAD
-  unsigned int __pyx_v_i;
-  struct __pyx_obj_5_cdec_Lattice *__pyx_v_self;
-  Py_ssize_t __pyx_t_0;
-  unsigned int __pyx_t_1;
-};
-
-
 /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":18
  *         self.vector[0][fid] = value
  * 
@@ -559,6 +629,39 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_2_kbest {
 };
 
 
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":13
+ *     return ret
+ * 
+ * cdef class Candidate:             # <<<<<<<<<<<<<<
+ *     cdef mteval.Candidate* candidate
+ *     cdef public float score
+ */
+struct __pyx_obj_5_cdec_Candidate {
+  PyObject_HEAD
+  const training::Candidate *candidate;
+  float score;
+};
+
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":69
+ *         del derivations
+ * 
+ *     def sample(self, unsigned n):             # <<<<<<<<<<<<<<
+ *         cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]()
+ *         if self.rng == NULL:
+ */
+struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample {
+  PyObject_HEAD
+  std::vector<HypergraphSampler::Hypothesis> *__pyx_v_hypos;
+  unsigned int __pyx_v_k;
+  unsigned int __pyx_v_n;
+  struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self;
+  PyObject *__pyx_v_sentence;
+  size_t __pyx_t_0;
+  unsigned int __pyx_t_1;
+};
+
+
 /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":3
  * from cython.operator cimport preincrement as pinc
  * 
@@ -571,6 +674,39 @@ struct __pyx_obj_5_cdec_DenseVector {
   std::vector<weight_t> *vector;
 };
 
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi":50
+ *         return hypergraph.AsPLF(self.lattice[0], True).c_str()
+ * 
+ *     def __iter__(self):             # <<<<<<<<<<<<<<
+ *         cdef unsigned i
+ *         for i in range(len(self)):
+ */
+struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ {
+  PyObject_HEAD
+  unsigned int __pyx_v_i;
+  struct __pyx_obj_5_cdec_Lattice *__pyx_v_self;
+  Py_ssize_t __pyx_t_0;
+  unsigned int __pyx_t_1;
+};
+
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":45
+ *         return self.stats.size()
+ * 
+ *     def __iter__(self):             # <<<<<<<<<<<<<<
+ *         for i in range(len(self)):
+ *             yield self.stats[0][i]
+ */
+struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ {
+  PyObject_HEAD
+  PyObject *__pyx_v_i;
+  struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self;
+  Py_ssize_t __pyx_t_0;
+  PyObject *__pyx_t_1;
+  PyObject *(*__pyx_t_2)(PyObject *);
+};
+
 #ifndef CYTHON_REFNANNY
   #define CYTHON_REFNANNY 0
 #endif
@@ -635,6 +771,8 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject
 static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed,
     const char *name, int exact); /*proto*/
 
+static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); /*proto*/
+
 static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); /*proto*/
 
 static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \
@@ -736,8 +874,6 @@ static double __Pyx__PyObject_AsDouble(PyObject* obj); /* proto */
 
 static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
 
-static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); /*proto*/
-
 static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb); /*proto*/
 static void __Pyx_ExceptionReset(PyObject *type, PyObject *value, PyObject *tb); /*proto*/
 
@@ -801,6 +937,9 @@ static __pyx_GeneratorObject *__Pyx_Generator_New(__pyx_generator_body_t body,
                                                   PyObject *closure);
 static int __pyx_Generator_init(void);
 
+static void __Pyx_WriteUnraisable(const char *name, int clineno,
+                                  int lineno, const char *filename); /*proto*/
+
 static int __Pyx_check_binary_version(void);
 
 typedef struct {
@@ -841,18 +980,28 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
 
 /* Module declarations from 'kbest' */
 
+/* Module declarations from 'mteval' */
+
 /* Module declarations from '_cdec' */
 static PyTypeObject *__pyx_ptype_5_cdec_DenseVector = 0;
 static PyTypeObject *__pyx_ptype_5_cdec_SparseVector = 0;
 static PyTypeObject *__pyx_ptype_5_cdec_Hypergraph = 0;
 static PyTypeObject *__pyx_ptype_5_cdec_Lattice = 0;
+static PyTypeObject *__pyx_ptype_5_cdec_Candidate = 0;
+static PyTypeObject *__pyx_ptype_5_cdec_SufficientStats = 0;
+static PyTypeObject *__pyx_ptype_5_cdec_SegmentEvaluator = 0;
+static PyTypeObject *__pyx_ptype_5_cdec_Scorer = 0;
 static PyTypeObject *__pyx_ptype_5_cdec_Decoder = 0;
 static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct____iter__ = 0;
 static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_1___iter__ = 0;
 static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_2_kbest = 0;
 static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_3_kbest_tree = 0;
-static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_4_sample = 0;
-static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_5___iter__ = 0;
+static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_4_kbest_features = 0;
+static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_5_sample = 0;
+static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_6___iter__ = 0;
+static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_7___iter__ = 0;
+static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_8_candidate_set = 0;
+static char *__pyx_f_5_cdec_as_str(PyObject *, struct __pyx_opt_args_5_cdec_as_str *__pyx_optional_args); /*proto*/
 #define __Pyx_MODULE_NAME "_cdec"
 int __pyx_module_is_main__cdec = 0;
 
@@ -860,11 +1009,10 @@ int __pyx_module_is_main__cdec = 0;
 static PyObject *__pyx_builtin_Exception;
 static PyObject *__pyx_builtin_KeyError;
 static PyObject *__pyx_builtin_range;
-static PyObject *__pyx_builtin_ValueError;
+static PyObject *__pyx_builtin_TypeError;
 static PyObject *__pyx_builtin_NotImplemented;
 static PyObject *__pyx_builtin_eval;
 static PyObject *__pyx_builtin_enumerate;
-static PyObject *__pyx_builtin_TypeError;
 static PyObject *__pyx_builtin_IndexError;
 static PyObject *__pyx_builtin_open;
 static PyObject *__pyx_pf_5_cdec_11DenseVector___getitem__(struct __pyx_obj_5_cdec_DenseVector *__pyx_v_self, char *__pyx_v_fname); /* proto */
@@ -877,7 +1025,7 @@ static int __pyx_pf_5_cdec_12SparseVector_2__setitem__(struct __pyx_obj_5_cdec_S
 static PyObject *__pyx_pf_5_cdec_12SparseVector_4__iter__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_12SparseVector_7dot(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, PyObject *__pyx_v_other); /* proto */
 static PyObject *__pyx_pf_5_cdec_12SparseVector_9todense(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5_cdec_12SparseVector_11__richcmp__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_other, int __pyx_v_op); /* proto */
+static PyObject *__pyx_pf_5_cdec_12SparseVector_11__richcmp__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_x, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_y, int __pyx_v_op); /* proto */
 static Py_ssize_t __pyx_pf_5_cdec_12SparseVector_13__len__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self); /* proto */
 static int __pyx_pf_5_cdec_12SparseVector_15__contains__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, char *__pyx_v_fname); /* proto */
 static PyObject *__pyx_pf_5_cdec_12SparseVector_17__iadd__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_other); /* proto */
@@ -886,8 +1034,12 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_21__imul__(struct __pyx_obj_5_cd
 #if PY_MAJOR_VERSION < 3
 static PyObject *__pyx_pf_5_cdec_12SparseVector_23__idiv__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, float __pyx_v_scalar); /* proto */
 #endif
-static PyObject *__pyx_pf_5_cdec_12SparseVector_25__add__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_other); /* proto */
-static PyObject *__pyx_pf_5_cdec_12SparseVector_27__sub__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_other); /* proto */
+static PyObject *__pyx_pf_5_cdec_12SparseVector_25__add__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_x, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_y); /* proto */
+static PyObject *__pyx_pf_5_cdec_12SparseVector_27__sub__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_x, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_y); /* proto */
+static PyObject *__pyx_pf_5_cdec_12SparseVector_29__mul__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /* proto */
+#if PY_MAJOR_VERSION < 3
+static PyObject *__pyx_pf_5_cdec_12SparseVector_31__div__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /* proto */
+#endif
 static void __pyx_pf_5_cdec_10Hypergraph___dealloc__(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_10Hypergraph_2viterbi(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_10Hypergraph_4viterbi_tree(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */
@@ -895,11 +1047,12 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_6viterbi_source_tree(struct __pyx_
 static PyObject *__pyx_pf_5_cdec_10Hypergraph_8viterbi_features(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_10Hypergraph_10kbest(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_size); /* proto */
 static PyObject *__pyx_pf_5_cdec_10Hypergraph_13kbest_tree(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_size); /* proto */
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_16sample(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, unsigned int __pyx_v_n); /* proto */
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_19intersect(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, struct __pyx_obj_5_cdec_Lattice *__pyx_v_lat); /* proto */
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_21prune(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_beam_alpha, PyObject *__pyx_v_density, PyObject *__pyx_v_kwargs); /* proto */
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_23lattice(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_25reweight(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_weights); /* proto */
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_16kbest_features(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_size); /* proto */
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_19sample(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, unsigned int __pyx_v_n); /* proto */
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_22intersect(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, struct __pyx_obj_5_cdec_Lattice *__pyx_v_lat); /* proto */
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_beam_alpha, PyObject *__pyx_v_density, PyObject *__pyx_v_kwargs); /* proto */
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_26lattice(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_weights); /* proto */
 static int __pyx_pf_5_cdec_7Lattice___init__(struct __pyx_obj_5_cdec_Lattice *__pyx_v_self, PyObject *__pyx_v_inp); /* proto */
 static PyObject *__pyx_pf_5_cdec_7Lattice_2__getitem__(struct __pyx_obj_5_cdec_Lattice *__pyx_v_self, int __pyx_v_index); /* proto */
 static int __pyx_pf_5_cdec_7Lattice_4__setitem__(struct __pyx_obj_5_cdec_Lattice *__pyx_v_self, int __pyx_v_index, PyObject *__pyx_v_arcs); /* proto */
@@ -907,6 +1060,23 @@ static Py_ssize_t __pyx_pf_5_cdec_7Lattice_6__len__(struct __pyx_obj_5_cdec_Latt
 static PyObject *__pyx_pf_5_cdec_7Lattice_8__str__(struct __pyx_obj_5_cdec_Lattice *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_7Lattice_10__iter__(struct __pyx_obj_5_cdec_Lattice *__pyx_v_self); /* proto */
 static void __pyx_pf_5_cdec_7Lattice_13__dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_Lattice *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_9Candidate_5words___get__(struct __pyx_obj_5_cdec_Candidate *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_9Candidate_4fmap___get__(struct __pyx_obj_5_cdec_Candidate *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_9Candidate_5score___get__(struct __pyx_obj_5_cdec_Candidate *__pyx_v_self); /* proto */
+static int __pyx_pf_5_cdec_9Candidate_5score_2__set__(struct __pyx_obj_5_cdec_Candidate *__pyx_v_self, PyObject *__pyx_v_value); /* proto */
+static void __pyx_pf_5_cdec_15SufficientStats___dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_5score___get__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_6detail___get__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self); /* proto */
+static Py_ssize_t __pyx_pf_5_cdec_15SufficientStats_2__len__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_4__iter__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__iadd__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self, struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_other); /* proto */
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__add__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_x, struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_y); /* proto */
+static void __pyx_pf_5_cdec_16SegmentEvaluator___dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self, PyObject *__pyx_v_sentence); /* proto */
+static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self, struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_hypergraph, unsigned int __pyx_v_k); /* proto */
+static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, char *__pyx_v_name); /* proto */
+static PyObject *__pyx_pf_5_cdec_6Scorer_2__call__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, PyObject *__pyx_v_refs); /* proto */
+static PyObject *__pyx_pf_5_cdec_6Scorer_4__str__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self); /* proto */
 static int __pyx_pf_5_cdec_7Decoder___cinit__(struct __pyx_obj_5_cdec_Decoder *__pyx_v_self, char *__pyx_v_config); /* proto */
 static void __pyx_pf_5_cdec_7Decoder_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_Decoder *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_Decoder *__pyx_v_self, PyObject *__pyx_v_cfg); /* proto */
@@ -920,12 +1090,18 @@ static char __pyx_k_10[] = "csplit_preserve_full_word";
 static char __pyx_k_11[] = "cannot reweight hypergraph with %s";
 static char __pyx_k_13[] = "Cannot create lattice from %s";
 static char __pyx_k_14[] = "lattice index out of range";
-static char __pyx_k_21[] = "Cannot translate input type %s";
+static char __pyx_k_19[] = "Cannot convert type %s to str";
+static char __pyx_k_23[] = "Cannot translate input type %s";
+static char __pyx_k__k[] = "k";
+static char __pyx_k__TER[] = "TER";
 static char __pyx_k__dot[] = "dot";
 static char __pyx_k__inp[] = "inp";
 static char __pyx_k__plf[] = "plf";
+static char __pyx_k__BLEU[] = "BLEU";
 static char __pyx_k__eval[] = "eval";
+static char __pyx_k__name[] = "name";
 static char __pyx_k__open[] = "open";
+static char __pyx_k__refs[] = "refs";
 static char __pyx_k__self[] = "self";
 static char __pyx_k__utf8[] = "utf8";
 static char __pyx_k___cdec[] = "_cdec";
@@ -937,18 +1113,20 @@ static char __pyx_k__decode[] = "decode";
 static char __pyx_k__encode[] = "encode";
 static char __pyx_k__density[] = "density";
 static char __pyx_k__grammar[] = "grammar";
+static char __pyx_k__IBM_BLEU[] = "IBM_BLEU";
 static char __pyx_k__KeyError[] = "KeyError";
 static char __pyx_k____exit__[] = "__exit__";
 static char __pyx_k____main__[] = "__main__";
 static char __pyx_k____test__[] = "__test__";
+static char __pyx_k__encoding[] = "encoding";
 static char __pyx_k__sentence[] = "sentence";
 static char __pyx_k__Exception[] = "Exception";
 static char __pyx_k__TypeError[] = "TypeError";
 static char __pyx_k____enter__[] = "__enter__";
 static char __pyx_k__enumerate[] = "enumerate";
 static char __pyx_k__IndexError[] = "IndexError";
-static char __pyx_k__ValueError[] = "ValueError";
 static char __pyx_k__beam_alpha[] = "beam_alpha";
+static char __pyx_k__hypergraph[] = "hypergraph";
 static char __pyx_k__ParseFailed[] = "ParseFailed";
 static char __pyx_k__NotImplemented[] = "NotImplemented";
 static PyObject *__pyx_kp_s_1;
@@ -956,15 +1134,18 @@ static PyObject *__pyx_n_s_10;
 static PyObject *__pyx_kp_s_11;
 static PyObject *__pyx_kp_s_13;
 static PyObject *__pyx_kp_s_14;
+static PyObject *__pyx_kp_s_19;
 static PyObject *__pyx_kp_s_2;
-static PyObject *__pyx_kp_s_21;
+static PyObject *__pyx_kp_s_23;
+static PyObject *__pyx_n_s__BLEU;
 static PyObject *__pyx_n_s__Exception;
+static PyObject *__pyx_n_s__IBM_BLEU;
 static PyObject *__pyx_n_s__IndexError;
 static PyObject *__pyx_n_s__KeyError;
 static PyObject *__pyx_n_s__NotImplemented;
 static PyObject *__pyx_n_s__ParseFailed;
+static PyObject *__pyx_n_s__TER;
 static PyObject *__pyx_n_s__TypeError;
-static PyObject *__pyx_n_s__ValueError;
 static PyObject *__pyx_n_s____enter__;
 static PyObject *__pyx_n_s____exit__;
 static PyObject *__pyx_n_s____main__;
@@ -976,13 +1157,18 @@ static PyObject *__pyx_n_s__decode;
 static PyObject *__pyx_n_s__density;
 static PyObject *__pyx_n_s__dot;
 static PyObject *__pyx_n_s__encode;
+static PyObject *__pyx_n_s__encoding;
 static PyObject *__pyx_n_s__enumerate;
 static PyObject *__pyx_n_s__eval;
 static PyObject *__pyx_n_s__grammar;
+static PyObject *__pyx_n_s__hypergraph;
 static PyObject *__pyx_n_s__inp;
+static PyObject *__pyx_n_s__k;
+static PyObject *__pyx_n_s__name;
 static PyObject *__pyx_n_s__open;
 static PyObject *__pyx_n_s__plf;
 static PyObject *__pyx_n_s__range;
+static PyObject *__pyx_n_s__refs;
 static PyObject *__pyx_n_s__self;
 static PyObject *__pyx_n_s__sentence;
 static PyObject *__pyx_n_s__split;
@@ -1002,8 +1188,11 @@ static PyObject *__pyx_k_tuple_15;
 static PyObject *__pyx_k_tuple_16;
 static PyObject *__pyx_k_tuple_17;
 static PyObject *__pyx_k_tuple_18;
-static PyObject *__pyx_k_tuple_19;
 static PyObject *__pyx_k_tuple_20;
+static PyObject *__pyx_k_tuple_21;
+static PyObject *__pyx_k_tuple_22;
+static PyObject *__pyx_k_tuple_24;
+static PyObject *__pyx_k_tuple_25;
 
 /* Python wrapper */
 static PyObject *__pyx_pw_5_cdec_11DenseVector_1__getitem__(PyObject *__pyx_v_self, PyObject *__pyx_arg_fname); /*proto*/
@@ -1857,7 +2046,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_7dot(struct __pyx_obj_5_cdec_Spa
  *             return self.vector.dot((<DenseVector> other).vector[0])
  *         elif isinstance(other, SparseVector):             # <<<<<<<<<<<<<<
  *             return self.vector.dot((<SparseVector> other).vector[0])
- *         raise ValueError('cannot take the dot product of %s and SparseVector' % type(other))
+ *         raise TypeError('cannot take the dot product of %s and SparseVector' % type(other))
  */
   __pyx_t_1 = ((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector));
   __Pyx_INCREF(__pyx_t_1);
@@ -1869,7 +2058,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_7dot(struct __pyx_obj_5_cdec_Spa
  *             return self.vector.dot((<DenseVector> other).vector[0])
  *         elif isinstance(other, SparseVector):
  *             return self.vector.dot((<SparseVector> other).vector[0])             # <<<<<<<<<<<<<<
- *         raise ValueError('cannot take the dot product of %s and SparseVector' % type(other))
+ *         raise TypeError('cannot take the dot product of %s and SparseVector' % type(other))
  * 
  */
     __Pyx_XDECREF(__pyx_r);
@@ -1885,7 +2074,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_7dot(struct __pyx_obj_5_cdec_Spa
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":56
  *         elif isinstance(other, SparseVector):
  *             return self.vector.dot((<SparseVector> other).vector[0])
- *         raise ValueError('cannot take the dot product of %s and SparseVector' % type(other))             # <<<<<<<<<<<<<<
+ *         raise TypeError('cannot take the dot product of %s and SparseVector' % type(other))             # <<<<<<<<<<<<<<
  * 
  *     def todense(self):
  */
@@ -1896,7 +2085,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_7dot(struct __pyx_obj_5_cdec_Spa
   PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_t_1));
   __Pyx_GIVEREF(((PyObject *)__pyx_t_1));
   __pyx_t_1 = 0;
-  __pyx_t_1 = PyObject_Call(__pyx_builtin_ValueError, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
   __Pyx_Raise(__pyx_t_1, 0, 0, 0);
@@ -1928,7 +2117,7 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_10todense(PyObject *__pyx_v_self
 }
 
 /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":58
- *         raise ValueError('cannot take the dot product of %s and SparseVector' % type(other))
+ *         raise TypeError('cannot take the dot product of %s and SparseVector' % type(other))
  * 
  *     def todense(self):             # <<<<<<<<<<<<<<
  *         cdef DenseVector dense = DenseVector()
@@ -1980,7 +2169,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_9todense(struct __pyx_obj_5_cdec
  *         self.vector.init_vector(dense.vector)
  *         return dense             # <<<<<<<<<<<<<<
  * 
- *     def __richcmp__(SparseVector self, SparseVector other, int op):
+ *     def __richcmp__(SparseVector x, SparseVector y, int op):
  */
   __Pyx_XDECREF(__pyx_r);
   __Pyx_INCREF(((PyObject *)__pyx_v_dense));
@@ -2001,14 +2190,14 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_9todense(struct __pyx_obj_5_cdec
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_12SparseVector_12__richcmp__(PyObject *__pyx_v_self, PyObject *__pyx_v_other, int __pyx_v_op); /*proto*/
-static PyObject *__pyx_pw_5_cdec_12SparseVector_12__richcmp__(PyObject *__pyx_v_self, PyObject *__pyx_v_other, int __pyx_v_op) {
+static PyObject *__pyx_pw_5_cdec_12SparseVector_12__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_v_op); /*proto*/
+static PyObject *__pyx_pw_5_cdec_12SparseVector_12__richcmp__(PyObject *__pyx_v_x, PyObject *__pyx_v_y, int __pyx_v_op) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__richcmp__ (wrapper)", 0);
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_self), __pyx_ptype_5_cdec_SparseVector, 1, "self", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5_cdec_SparseVector, 1, "other", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_r = __pyx_pf_5_cdec_12SparseVector_11__richcmp__(((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_self), ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_other), ((int)__pyx_v_op));
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_ptype_5_cdec_SparseVector, 1, "x", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_ptype_5_cdec_SparseVector, 1, "y", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5_cdec_12SparseVector_11__richcmp__(((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_x), ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_y), ((int)__pyx_v_op));
   goto __pyx_L0;
   __pyx_L1_error:;
   __pyx_r = NULL;
@@ -2020,12 +2209,12 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_12__richcmp__(PyObject *__pyx_v_
 /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":64
  *         return dense
  * 
- *     def __richcmp__(SparseVector self, SparseVector other, int op):             # <<<<<<<<<<<<<<
+ *     def __richcmp__(SparseVector x, SparseVector y, int op):             # <<<<<<<<<<<<<<
  *         if op == 2: # ==
- *             return self.vector[0] == other.vector[0]
+ *             return x.vector[0] == y.vector[0]
  */
 
-static PyObject *__pyx_pf_5_cdec_12SparseVector_11__richcmp__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_other, int __pyx_v_op) {
+static PyObject *__pyx_pf_5_cdec_12SparseVector_11__richcmp__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_x, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_y, int __pyx_v_op) {
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
   PyObject *__pyx_t_1 = NULL;
@@ -2037,31 +2226,31 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_11__richcmp__(struct __pyx_obj_5
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":67
  *         if op == 2: # ==
- *             return self.vector[0] == other.vector[0]
+ *             return x.vector[0] == y.vector[0]
  *         elif op == 3: # !=             # <<<<<<<<<<<<<<
- *             return not (self == other)
+ *             return not (x == y)
  *         raise NotImplemented('comparison not implemented for SparseVector')
  */
   switch (__pyx_v_op) {
 
     /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":65
  * 
- *     def __richcmp__(SparseVector self, SparseVector other, int op):
+ *     def __richcmp__(SparseVector x, SparseVector y, int op):
  *         if op == 2: # ==             # <<<<<<<<<<<<<<
- *             return self.vector[0] == other.vector[0]
+ *             return x.vector[0] == y.vector[0]
  *         elif op == 3: # !=
  */
     case 2:
 
     /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":66
- *     def __richcmp__(SparseVector self, SparseVector other, int op):
+ *     def __richcmp__(SparseVector x, SparseVector y, int op):
  *         if op == 2: # ==
- *             return self.vector[0] == other.vector[0]             # <<<<<<<<<<<<<<
+ *             return x.vector[0] == y.vector[0]             # <<<<<<<<<<<<<<
  *         elif op == 3: # !=
- *             return not (self == other)
+ *             return not (x == y)
  */
     __Pyx_XDECREF(__pyx_r);
-    __pyx_t_1 = __Pyx_PyBool_FromLong(((__pyx_v_self->vector[0]) == (__pyx_v_other->vector[0]))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = __Pyx_PyBool_FromLong(((__pyx_v_x->vector[0]) == (__pyx_v_y->vector[0]))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_1);
     __pyx_r = __pyx_t_1;
     __pyx_t_1 = 0;
@@ -2070,22 +2259,22 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_11__richcmp__(struct __pyx_obj_5
 
     /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":67
  *         if op == 2: # ==
- *             return self.vector[0] == other.vector[0]
+ *             return x.vector[0] == y.vector[0]
  *         elif op == 3: # !=             # <<<<<<<<<<<<<<
- *             return not (self == other)
+ *             return not (x == y)
  *         raise NotImplemented('comparison not implemented for SparseVector')
  */
     case 3:
 
     /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":68
- *             return self.vector[0] == other.vector[0]
+ *             return x.vector[0] == y.vector[0]
  *         elif op == 3: # !=
- *             return not (self == other)             # <<<<<<<<<<<<<<
+ *             return not (x == y)             # <<<<<<<<<<<<<<
  *         raise NotImplemented('comparison not implemented for SparseVector')
  * 
  */
     __Pyx_XDECREF(__pyx_r);
-    __pyx_t_1 = PyObject_RichCompare(((PyObject *)__pyx_v_self), ((PyObject *)__pyx_v_other), Py_EQ); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = PyObject_RichCompare(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y), Py_EQ); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_1);
     __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_2 < 0)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
@@ -2099,7 +2288,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_11__richcmp__(struct __pyx_obj_5
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":69
  *         elif op == 3: # !=
- *             return not (self == other)
+ *             return not (x == y)
  *         raise NotImplemented('comparison not implemented for SparseVector')             # <<<<<<<<<<<<<<
  * 
  *     def __len__(self):
@@ -2439,7 +2628,7 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_23__idiv__(struct __pyx_obj_5_cd
  *         self.vector[0] /= scalar
  *         return self             # <<<<<<<<<<<<<<
  * 
- *     def __add__(SparseVector self, SparseVector other):
+ *     def __add__(SparseVector x, SparseVector y):
  */
   __Pyx_XDECREF(__pyx_r);
   __Pyx_INCREF(((PyObject *)__pyx_v_self));
@@ -2455,14 +2644,14 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_23__idiv__(struct __pyx_obj_5_cd
 #endif /*!(#if PY_MAJOR_VERSION < 3)*/
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_12SparseVector_26__add__(PyObject *__pyx_v_self, PyObject *__pyx_v_other); /*proto*/
-static PyObject *__pyx_pw_5_cdec_12SparseVector_26__add__(PyObject *__pyx_v_self, PyObject *__pyx_v_other) {
+static PyObject *__pyx_pw_5_cdec_12SparseVector_26__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /*proto*/
+static PyObject *__pyx_pw_5_cdec_12SparseVector_26__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__add__ (wrapper)", 0);
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_self), __pyx_ptype_5_cdec_SparseVector, 1, "self", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5_cdec_SparseVector, 1, "other", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_r = __pyx_pf_5_cdec_12SparseVector_25__add__(((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_self), ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_other));
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_ptype_5_cdec_SparseVector, 1, "x", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_ptype_5_cdec_SparseVector, 1, "y", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5_cdec_12SparseVector_25__add__(((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_x), ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_y));
   goto __pyx_L0;
   __pyx_L1_error:;
   __pyx_r = NULL;
@@ -2474,12 +2663,12 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_26__add__(PyObject *__pyx_v_self
 /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":93
  *         return self
  * 
- *     def __add__(SparseVector self, SparseVector other):             # <<<<<<<<<<<<<<
+ *     def __add__(SparseVector x, SparseVector y):             # <<<<<<<<<<<<<<
  *         cdef SparseVector result = SparseVector()
- *         result.vector = new FastSparseVector[weight_t](self.vector[0] + other.vector[0])
+ *         result.vector = new FastSparseVector[weight_t](x.vector[0] + y.vector[0])
  */
 
-static PyObject *__pyx_pf_5_cdec_12SparseVector_25__add__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_other) {
+static PyObject *__pyx_pf_5_cdec_12SparseVector_25__add__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_x, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_y) {
   struct __pyx_obj_5_cdec_SparseVector *__pyx_v_result = 0;
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
@@ -2491,9 +2680,9 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_25__add__(struct __pyx_obj_5_cde
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":94
  * 
- *     def __add__(SparseVector self, SparseVector other):
+ *     def __add__(SparseVector x, SparseVector y):
  *         cdef SparseVector result = SparseVector()             # <<<<<<<<<<<<<<
- *         result.vector = new FastSparseVector[weight_t](self.vector[0] + other.vector[0])
+ *         result.vector = new FastSparseVector[weight_t](x.vector[0] + y.vector[0])
  *         return result
  */
   __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
@@ -2502,20 +2691,20 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_25__add__(struct __pyx_obj_5_cde
   __pyx_t_1 = 0;
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":95
- *     def __add__(SparseVector self, SparseVector other):
+ *     def __add__(SparseVector x, SparseVector y):
  *         cdef SparseVector result = SparseVector()
- *         result.vector = new FastSparseVector[weight_t](self.vector[0] + other.vector[0])             # <<<<<<<<<<<<<<
+ *         result.vector = new FastSparseVector[weight_t](x.vector[0] + y.vector[0])             # <<<<<<<<<<<<<<
  *         return result
  * 
  */
-  __pyx_v_result->vector = new FastSparseVector<weight_t>(((__pyx_v_self->vector[0]) + (__pyx_v_other->vector[0])));
+  __pyx_v_result->vector = new FastSparseVector<weight_t>(((__pyx_v_x->vector[0]) + (__pyx_v_y->vector[0])));
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":96
  *         cdef SparseVector result = SparseVector()
- *         result.vector = new FastSparseVector[weight_t](self.vector[0] + other.vector[0])
+ *         result.vector = new FastSparseVector[weight_t](x.vector[0] + y.vector[0])
  *         return result             # <<<<<<<<<<<<<<
  * 
- *     def __sub__(SparseVector self, SparseVector other):
+ *     def __sub__(SparseVector x, SparseVector y):
  */
   __Pyx_XDECREF(__pyx_r);
   __Pyx_INCREF(((PyObject *)__pyx_v_result));
@@ -2536,14 +2725,14 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_25__add__(struct __pyx_obj_5_cde
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_12SparseVector_28__sub__(PyObject *__pyx_v_self, PyObject *__pyx_v_other); /*proto*/
-static PyObject *__pyx_pw_5_cdec_12SparseVector_28__sub__(PyObject *__pyx_v_self, PyObject *__pyx_v_other) {
+static PyObject *__pyx_pw_5_cdec_12SparseVector_28__sub__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /*proto*/
+static PyObject *__pyx_pw_5_cdec_12SparseVector_28__sub__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__sub__ (wrapper)", 0);
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_self), __pyx_ptype_5_cdec_SparseVector, 1, "self", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5_cdec_SparseVector, 1, "other", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_r = __pyx_pf_5_cdec_12SparseVector_27__sub__(((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_self), ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_other));
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_ptype_5_cdec_SparseVector, 1, "x", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_ptype_5_cdec_SparseVector, 1, "y", 0))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5_cdec_12SparseVector_27__sub__(((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_x), ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_v_y));
   goto __pyx_L0;
   __pyx_L1_error:;
   __pyx_r = NULL;
@@ -2555,12 +2744,12 @@ static PyObject *__pyx_pw_5_cdec_12SparseVector_28__sub__(PyObject *__pyx_v_self
 /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":98
  *         return result
  * 
- *     def __sub__(SparseVector self, SparseVector other):             # <<<<<<<<<<<<<<
+ *     def __sub__(SparseVector x, SparseVector y):             # <<<<<<<<<<<<<<
  *         cdef SparseVector result = SparseVector()
- *         result.vector = new FastSparseVector[weight_t](self.vector[0] - other.vector[0])
+ *         result.vector = new FastSparseVector[weight_t](x.vector[0] - y.vector[0])
  */
 
-static PyObject *__pyx_pf_5_cdec_12SparseVector_27__sub__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_self, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_other) {
+static PyObject *__pyx_pf_5_cdec_12SparseVector_27__sub__(struct __pyx_obj_5_cdec_SparseVector *__pyx_v_x, struct __pyx_obj_5_cdec_SparseVector *__pyx_v_y) {
   struct __pyx_obj_5_cdec_SparseVector *__pyx_v_result = 0;
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
@@ -2572,9 +2761,9 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_27__sub__(struct __pyx_obj_5_cde
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":99
  * 
- *     def __sub__(SparseVector self, SparseVector other):
+ *     def __sub__(SparseVector x, SparseVector y):
  *         cdef SparseVector result = SparseVector()             # <<<<<<<<<<<<<<
- *         result.vector = new FastSparseVector[weight_t](self.vector[0] - other.vector[0])
+ *         result.vector = new FastSparseVector[weight_t](x.vector[0] - y.vector[0])
  *         return result
  */
   __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
@@ -2583,17 +2772,20 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_27__sub__(struct __pyx_obj_5_cde
   __pyx_t_1 = 0;
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":100
- *     def __sub__(SparseVector self, SparseVector other):
+ *     def __sub__(SparseVector x, SparseVector y):
  *         cdef SparseVector result = SparseVector()
- *         result.vector = new FastSparseVector[weight_t](self.vector[0] - other.vector[0])             # <<<<<<<<<<<<<<
+ *         result.vector = new FastSparseVector[weight_t](x.vector[0] - y.vector[0])             # <<<<<<<<<<<<<<
  *         return result
+ * 
  */
-  __pyx_v_result->vector = new FastSparseVector<weight_t>(((__pyx_v_self->vector[0]) - (__pyx_v_other->vector[0])));
+  __pyx_v_result->vector = new FastSparseVector<weight_t>(((__pyx_v_x->vector[0]) - (__pyx_v_y->vector[0])));
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":101
  *         cdef SparseVector result = SparseVector()
- *         result.vector = new FastSparseVector[weight_t](self.vector[0] - other.vector[0])
+ *         result.vector = new FastSparseVector[weight_t](x.vector[0] - y.vector[0])
  *         return result             # <<<<<<<<<<<<<<
+ * 
+ *     def __mul__(x, y):
  */
   __Pyx_XDECREF(__pyx_r);
   __Pyx_INCREF(((PyObject *)__pyx_v_result));
@@ -2614,6 +2806,247 @@ static PyObject *__pyx_pf_5_cdec_12SparseVector_27__sub__(struct __pyx_obj_5_cde
 }
 
 /* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_12SparseVector_30__mul__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /*proto*/
+static PyObject *__pyx_pw_5_cdec_12SparseVector_30__mul__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__mul__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_12SparseVector_29__mul__(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":103
+ *         return result
+ * 
+ *     def __mul__(x, y):             # <<<<<<<<<<<<<<
+ *         cdef SparseVector vector
+ *         cdef float scalar
+ */
+
+static PyObject *__pyx_pf_5_cdec_12SparseVector_29__mul__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
+  struct __pyx_obj_5_cdec_SparseVector *__pyx_v_vector = 0;
+  float __pyx_v_scalar;
+  struct __pyx_obj_5_cdec_SparseVector *__pyx_v_result = 0;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_t_2;
+  float __pyx_t_3;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__mul__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":106
+ *         cdef SparseVector vector
+ *         cdef float scalar
+ *         if isinstance(x, SparseVector): vector, scalar = x, y             # <<<<<<<<<<<<<<
+ *         else: vector, scalar = y, x
+ *         cdef SparseVector result = SparseVector()
+ */
+  __pyx_t_1 = ((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector));
+  __Pyx_INCREF(__pyx_t_1);
+  __pyx_t_2 = __Pyx_TypeCheck(__pyx_v_x, __pyx_t_1); 
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  if (__pyx_t_2) {
+    if (!(likely(((__pyx_v_x) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_x, __pyx_ptype_5_cdec_SparseVector))))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = __pyx_v_x;
+    __Pyx_INCREF(__pyx_t_1);
+    __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_y); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_v_vector = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1);
+    __pyx_t_1 = 0;
+    __pyx_v_scalar = __pyx_t_3;
+    goto __pyx_L3;
+  }
+  /*else*/ {
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":107
+ *         cdef float scalar
+ *         if isinstance(x, SparseVector): vector, scalar = x, y
+ *         else: vector, scalar = y, x             # <<<<<<<<<<<<<<
+ *         cdef SparseVector result = SparseVector()
+ *         result.vector = new FastSparseVector[weight_t](vector.vector[0] * scalar)
+ */
+    if (!(likely(((__pyx_v_y) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_y, __pyx_ptype_5_cdec_SparseVector))))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = __pyx_v_y;
+    __Pyx_INCREF(__pyx_t_1);
+    __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_x); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_v_vector = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1);
+    __pyx_t_1 = 0;
+    __pyx_v_scalar = __pyx_t_3;
+  }
+  __pyx_L3:;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":108
+ *         if isinstance(x, SparseVector): vector, scalar = x, y
+ *         else: vector, scalar = y, x
+ *         cdef SparseVector result = SparseVector()             # <<<<<<<<<<<<<<
+ *         result.vector = new FastSparseVector[weight_t](vector.vector[0] * scalar)
+ *         return result
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_result = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1);
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":109
+ *         else: vector, scalar = y, x
+ *         cdef SparseVector result = SparseVector()
+ *         result.vector = new FastSparseVector[weight_t](vector.vector[0] * scalar)             # <<<<<<<<<<<<<<
+ *         return result
+ * 
+ */
+  __pyx_v_result->vector = new FastSparseVector<weight_t>(((__pyx_v_vector->vector[0]) * __pyx_v_scalar));
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":110
+ *         cdef SparseVector result = SparseVector()
+ *         result.vector = new FastSparseVector[weight_t](vector.vector[0] * scalar)
+ *         return result             # <<<<<<<<<<<<<<
+ * 
+ *     def __div__(x, y):
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __Pyx_INCREF(((PyObject *)__pyx_v_result));
+  __pyx_r = ((PyObject *)__pyx_v_result);
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.SparseVector.__mul__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XDECREF((PyObject *)__pyx_v_vector);
+  __Pyx_XDECREF((PyObject *)__pyx_v_result);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+#if PY_MAJOR_VERSION < 3
+static PyObject *__pyx_pw_5_cdec_12SparseVector_32__div__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /*proto*/
+static PyObject *__pyx_pw_5_cdec_12SparseVector_32__div__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__div__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_12SparseVector_31__div__(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+#endif /*!(#if PY_MAJOR_VERSION < 3)*/
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":112
+ *         return result
+ * 
+ *     def __div__(x, y):             # <<<<<<<<<<<<<<
+ *         cdef SparseVector vector
+ *         cdef float scalar
+ */
+
+#if PY_MAJOR_VERSION < 3
+static PyObject *__pyx_pf_5_cdec_12SparseVector_31__div__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
+  struct __pyx_obj_5_cdec_SparseVector *__pyx_v_vector = 0;
+  float __pyx_v_scalar;
+  struct __pyx_obj_5_cdec_SparseVector *__pyx_v_result = 0;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_t_2;
+  float __pyx_t_3;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__div__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":115
+ *         cdef SparseVector vector
+ *         cdef float scalar
+ *         if isinstance(x, SparseVector): vector, scalar = x, y             # <<<<<<<<<<<<<<
+ *         else: vector, scalar = y, x
+ *         cdef SparseVector result = SparseVector()
+ */
+  __pyx_t_1 = ((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector));
+  __Pyx_INCREF(__pyx_t_1);
+  __pyx_t_2 = __Pyx_TypeCheck(__pyx_v_x, __pyx_t_1); 
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  if (__pyx_t_2) {
+    if (!(likely(((__pyx_v_x) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_x, __pyx_ptype_5_cdec_SparseVector))))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = __pyx_v_x;
+    __Pyx_INCREF(__pyx_t_1);
+    __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_y); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_v_vector = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1);
+    __pyx_t_1 = 0;
+    __pyx_v_scalar = __pyx_t_3;
+    goto __pyx_L3;
+  }
+  /*else*/ {
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":116
+ *         cdef float scalar
+ *         if isinstance(x, SparseVector): vector, scalar = x, y
+ *         else: vector, scalar = y, x             # <<<<<<<<<<<<<<
+ *         cdef SparseVector result = SparseVector()
+ *         result.vector = new FastSparseVector[weight_t](vector.vector[0] / scalar)
+ */
+    if (!(likely(((__pyx_v_y) == Py_None) || likely(__Pyx_TypeTest(__pyx_v_y, __pyx_ptype_5_cdec_SparseVector))))) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = __pyx_v_y;
+    __Pyx_INCREF(__pyx_t_1);
+    __pyx_t_3 = __pyx_PyFloat_AsFloat(__pyx_v_x); if (unlikely((__pyx_t_3 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_v_vector = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1);
+    __pyx_t_1 = 0;
+    __pyx_v_scalar = __pyx_t_3;
+  }
+  __pyx_L3:;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":117
+ *         if isinstance(x, SparseVector): vector, scalar = x, y
+ *         else: vector, scalar = y, x
+ *         cdef SparseVector result = SparseVector()             # <<<<<<<<<<<<<<
+ *         result.vector = new FastSparseVector[weight_t](vector.vector[0] / scalar)
+ *         return result
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_result = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1);
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":118
+ *         else: vector, scalar = y, x
+ *         cdef SparseVector result = SparseVector()
+ *         result.vector = new FastSparseVector[weight_t](vector.vector[0] / scalar)             # <<<<<<<<<<<<<<
+ *         return result
+ */
+  __pyx_v_result->vector = new FastSparseVector<weight_t>(((__pyx_v_vector->vector[0]) / __pyx_v_scalar));
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":119
+ *         cdef SparseVector result = SparseVector()
+ *         result.vector = new FastSparseVector[weight_t](vector.vector[0] / scalar)
+ *         return result             # <<<<<<<<<<<<<<
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __Pyx_INCREF(((PyObject *)__pyx_v_result));
+  __pyx_r = ((PyObject *)__pyx_v_result);
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.SparseVector.__div__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XDECREF((PyObject *)__pyx_v_vector);
+  __Pyx_XDECREF((PyObject *)__pyx_v_result);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+#endif /*!(#if PY_MAJOR_VERSION < 3)*/
+
+/* Python wrapper */
 static void __pyx_pw_5_cdec_10Hypergraph_1__dealloc__(PyObject *__pyx_v_self); /*proto*/
 static void __pyx_pw_5_cdec_10Hypergraph_1__dealloc__(PyObject *__pyx_v_self) {
   __Pyx_RefNannyDeclarations
@@ -3338,7 +3771,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator3(__pyx_GeneratorObject
  *             yield tree.decode('utf8')
  *         del derivations             # <<<<<<<<<<<<<<
  * 
- *     def sample(self, unsigned n):
+ *     def kbest_features(self, size):
  */
   delete __pyx_cur_scope->__pyx_v_derivations;
   PyErr_SetNone(PyExc_StopIteration);
@@ -3356,14 +3789,204 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_15generator3(__pyx_GeneratorObject
 static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_17sample(PyObject *__pyx_v_self, PyObject *__pyx_arg_n); /*proto*/
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_17sample(PyObject *__pyx_v_self, PyObject *__pyx_arg_n) {
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_17kbest_features(PyObject *__pyx_v_self, PyObject *__pyx_v_size); /*proto*/
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_17kbest_features(PyObject *__pyx_v_self, PyObject *__pyx_v_size) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("kbest_features (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_16kbest_features(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((PyObject *)__pyx_v_size));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":56
+ *         del derivations
+ * 
+ *     def kbest_features(self, size):             # <<<<<<<<<<<<<<
+ *         cdef kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal]* derivations = new kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal](self.hg[0], size)
+ *         cdef kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal].Derivation* derivation
+ */
+
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_16kbest_features(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_size) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *__pyx_cur_scope;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("kbest_features", 0);
+  __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *)__pyx_ptype_5_cdec___pyx_scope_struct_4_kbest_features->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_4_kbest_features, __pyx_empty_tuple, NULL);
+  if (unlikely(!__pyx_cur_scope)) {
+    __Pyx_RefNannyFinishContext();
+    return NULL;
+  }
+  __Pyx_GOTREF(__pyx_cur_scope);
+  __pyx_cur_scope->__pyx_v_self = __pyx_v_self;
+  __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
+  __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
+  __pyx_cur_scope->__pyx_v_size = __pyx_v_size;
+  __Pyx_INCREF(__pyx_cur_scope->__pyx_v_size);
+  __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_size);
+  {
+    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_10Hypergraph_18generator4, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_cur_scope);
+    __Pyx_RefNannyFinishContext();
+    return (PyObject *) gen;
+  }
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_AddTraceback("_cdec.Hypergraph.kbest_features", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(((PyObject *)__pyx_cur_scope));
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
+{
+  struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *)__pyx_generator->closure);
+  PyObject *__pyx_r = NULL;
+  unsigned int __pyx_t_1;
+  long __pyx_t_2;
+  int __pyx_t_3;
+  PyObject *__pyx_t_4 = NULL;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("None", 0);
+  switch (__pyx_generator->resume_label) {
+    case 0: goto __pyx_L3_first_run;
+    case 1: goto __pyx_L7_resume_from_yield;
+    default: /* CPython raises the right error here */
+    __Pyx_RefNannyFinishContext();
+    return NULL;
+  }
+  __pyx_L3_first_run:;
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":57
+ * 
+ *     def kbest_features(self, size):
+ *         cdef kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal]* derivations = new kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal](self.hg[0], size)             # <<<<<<<<<<<<<<
+ *         cdef kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal].Derivation* derivation
+ *         cdef SparseVector fmap
+ */
+  __pyx_t_1 = __Pyx_PyInt_AsUnsignedInt(__pyx_cur_scope->__pyx_v_size); if (unlikely((__pyx_t_1 == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 57; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_cur_scope->__pyx_v_derivations = new KBest::KBestDerivations<FastSparseVector<weight_t>,FeatureVectorTraversal>((__pyx_cur_scope->__pyx_v_self->hg[0]), __pyx_t_1);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":61
+ *         cdef SparseVector fmap
+ *         cdef unsigned k
+ *         for k in range(size):             # <<<<<<<<<<<<<<
+ *             derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k)
+ *             if not derivation: break
+ */
+  __pyx_t_2 = __Pyx_PyInt_AsLong(__pyx_cur_scope->__pyx_v_size); if (unlikely((__pyx_t_2 == (long)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  for (__pyx_t_1 = 0; __pyx_t_1 < __pyx_t_2; __pyx_t_1+=1) {
+    __pyx_cur_scope->__pyx_v_k = __pyx_t_1;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":62
+ *         cdef unsigned k
+ *         for k in range(size):
+ *             derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k)             # <<<<<<<<<<<<<<
+ *             if not derivation: break
+ *             fmap = SparseVector()
+ */
+    __pyx_cur_scope->__pyx_v_derivation = __pyx_cur_scope->__pyx_v_derivations->LazyKthBest((__pyx_cur_scope->__pyx_v_self->hg->nodes_.size() - 1), __pyx_cur_scope->__pyx_v_k);
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":63
+ *         for k in range(size):
+ *             derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k)
+ *             if not derivation: break             # <<<<<<<<<<<<<<
+ *             fmap = SparseVector()
+ *             fmap.vector = new FastSparseVector[weight_t](derivation._yield)
+ */
+    __pyx_t_3 = (!(__pyx_cur_scope->__pyx_v_derivation != 0));
+    if (__pyx_t_3) {
+      goto __pyx_L5_break;
+      goto __pyx_L6;
+    }
+    __pyx_L6:;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":64
+ *             derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k)
+ *             if not derivation: break
+ *             fmap = SparseVector()             # <<<<<<<<<<<<<<
+ *             fmap.vector = new FastSparseVector[weight_t](derivation._yield)
+ *             yield fmap
+ */
+    __pyx_t_4 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_4);
+    __Pyx_XGOTREF(((PyObject *)__pyx_cur_scope->__pyx_v_fmap));
+    __Pyx_XDECREF(((PyObject *)__pyx_cur_scope->__pyx_v_fmap));
+    __Pyx_GIVEREF(__pyx_t_4);
+    __pyx_cur_scope->__pyx_v_fmap = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_4);
+    __pyx_t_4 = 0;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":65
+ *             if not derivation: break
+ *             fmap = SparseVector()
+ *             fmap.vector = new FastSparseVector[weight_t](derivation._yield)             # <<<<<<<<<<<<<<
+ *             yield fmap
+ *         del derivations
+ */
+    __pyx_cur_scope->__pyx_v_fmap->vector = new FastSparseVector<weight_t>(__pyx_cur_scope->__pyx_v_derivation->yield);
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":66
+ *             fmap = SparseVector()
+ *             fmap.vector = new FastSparseVector[weight_t](derivation._yield)
+ *             yield fmap             # <<<<<<<<<<<<<<
+ *         del derivations
+ * 
+ */
+    __Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_v_fmap));
+    __pyx_r = ((PyObject *)__pyx_cur_scope->__pyx_v_fmap);
+    __pyx_cur_scope->__pyx_t_0 = __pyx_t_1;
+    __pyx_cur_scope->__pyx_t_1 = __pyx_t_2;
+    __Pyx_XGIVEREF(__pyx_r);
+    __Pyx_RefNannyFinishContext();
+    /* return from generator, yielding value */
+    __pyx_generator->resume_label = 1;
+    return __pyx_r;
+    __pyx_L7_resume_from_yield:;
+    __pyx_t_1 = __pyx_cur_scope->__pyx_t_0;
+    __pyx_t_2 = __pyx_cur_scope->__pyx_t_1;
+    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
+  __pyx_L5_break:;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":67
+ *             fmap.vector = new FastSparseVector[weight_t](derivation._yield)
+ *             yield fmap
+ *         del derivations             # <<<<<<<<<<<<<<
+ * 
+ *     def sample(self, unsigned n):
+ */
+  delete __pyx_cur_scope->__pyx_v_derivations;
+  PyErr_SetNone(PyExc_StopIteration);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_4);
+  __Pyx_AddTraceback("kbest_features", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_L0:;
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_generator->resume_label = -1;
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+}
+static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator5(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_20sample(PyObject *__pyx_v_self, PyObject *__pyx_arg_n); /*proto*/
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_20sample(PyObject *__pyx_v_self, PyObject *__pyx_arg_n) {
   unsigned int __pyx_v_n;
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("sample (wrapper)", 0);
   assert(__pyx_arg_n); {
-    __pyx_v_n = __Pyx_PyInt_AsUnsignedInt(__pyx_arg_n); if (unlikely((__pyx_v_n == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    __pyx_v_n = __Pyx_PyInt_AsUnsignedInt(__pyx_arg_n); if (unlikely((__pyx_v_n == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L3_error:;
@@ -3371,12 +3994,12 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_17sample(PyObject *__pyx_v_self, P
   __Pyx_RefNannyFinishContext();
   return NULL;
   __pyx_L4_argument_unpacking_done:;
-  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_16sample(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((unsigned int)__pyx_v_n));
+  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_19sample(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((unsigned int)__pyx_v_n));
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":56
+/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":69
  *         del derivations
  * 
  *     def sample(self, unsigned n):             # <<<<<<<<<<<<<<
@@ -3384,15 +4007,15 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_17sample(PyObject *__pyx_v_self, P
  *         if self.rng == NULL:
  */
 
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_16sample(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, unsigned int __pyx_v_n) {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *__pyx_cur_scope;
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_19sample(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, unsigned int __pyx_v_n) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *__pyx_cur_scope;
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
   int __pyx_lineno = 0;
   const char *__pyx_filename = NULL;
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("sample", 0);
-  __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *)__pyx_ptype_5_cdec___pyx_scope_struct_4_sample->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_4_sample, __pyx_empty_tuple, NULL);
+  __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *)__pyx_ptype_5_cdec___pyx_scope_struct_5_sample->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_5_sample, __pyx_empty_tuple, NULL);
   if (unlikely(!__pyx_cur_scope)) {
     __Pyx_RefNannyFinishContext();
     return NULL;
@@ -3403,7 +4026,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_16sample(struct __pyx_obj_5_cdec_H
   __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
   __pyx_cur_scope->__pyx_v_n = __pyx_v_n;
   {
-    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_10Hypergraph_18generator4, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_10Hypergraph_21generator5, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_DECREF(__pyx_cur_scope);
     __Pyx_RefNannyFinishContext();
     return (PyObject *) gen;
@@ -3421,9 +4044,9 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_16sample(struct __pyx_obj_5_cdec_H
   return __pyx_r;
 }
 
-static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
+static PyObject *__pyx_gb_5_cdec_10Hypergraph_21generator5(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
 {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *)__pyx_generator->closure);
+  struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *)__pyx_generator->closure);
   PyObject *__pyx_r = NULL;
   int __pyx_t_1;
   size_t __pyx_t_2;
@@ -3440,9 +4063,9 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject
     return NULL;
   }
   __pyx_L3_first_run:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":57
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":70
  * 
  *     def sample(self, unsigned n):
  *         cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]()             # <<<<<<<<<<<<<<
@@ -3451,7 +4074,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject
  */
   __pyx_cur_scope->__pyx_v_hypos = new std::vector<HypergraphSampler::Hypothesis>();
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":58
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":71
  *     def sample(self, unsigned n):
  *         cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]()
  *         if self.rng == NULL:             # <<<<<<<<<<<<<<
@@ -3461,7 +4084,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject
   __pyx_t_1 = (__pyx_cur_scope->__pyx_v_self->rng == NULL);
   if (__pyx_t_1) {
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":59
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":72
  *         cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]()
  *         if self.rng == NULL:
  *             self.rng = new MT19937()             # <<<<<<<<<<<<<<
@@ -3473,7 +4096,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject
   }
   __pyx_L4:;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":60
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":73
  *         if self.rng == NULL:
  *             self.rng = new MT19937()
  *         hypergraph.sample_hypotheses(self.hg[0], n, self.rng, hypos)             # <<<<<<<<<<<<<<
@@ -3482,7 +4105,7 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject
  */
   HypergraphSampler::sample_hypotheses((__pyx_cur_scope->__pyx_v_self->hg[0]), __pyx_cur_scope->__pyx_v_n, __pyx_cur_scope->__pyx_v_self->rng, __pyx_cur_scope->__pyx_v_hypos);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":63
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":76
  *         cdef str sentence
  *         cdef unsigned k
  *         for k in range(hypos.size()):             # <<<<<<<<<<<<<<
@@ -3493,32 +4116,32 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject
   for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) {
     __pyx_cur_scope->__pyx_v_k = __pyx_t_3;
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":64
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":77
  *         cdef unsigned k
  *         for k in range(hypos.size()):
  *             sentence = GetString(hypos[0][k].words).c_str()             # <<<<<<<<<<<<<<
  *             yield sentence.decode('utf8')
  *         del hypos
  */
-    __pyx_t_4 = PyBytes_FromString(TD::GetString(((__pyx_cur_scope->__pyx_v_hypos[0])[__pyx_cur_scope->__pyx_v_k]).words).c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = PyBytes_FromString(TD::GetString(((__pyx_cur_scope->__pyx_v_hypos[0])[__pyx_cur_scope->__pyx_v_k]).words).c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(((PyObject *)__pyx_t_4));
-    if (!(likely(PyString_CheckExact(((PyObject *)__pyx_t_4)))||(PyErr_Format(PyExc_TypeError, "Expected str, got %.200s", Py_TYPE(((PyObject *)__pyx_t_4))->tp_name), 0))) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    if (!(likely(PyString_CheckExact(((PyObject *)__pyx_t_4)))||(PyErr_Format(PyExc_TypeError, "Expected str, got %.200s", Py_TYPE(((PyObject *)__pyx_t_4))->tp_name), 0))) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_XGOTREF(((PyObject *)__pyx_cur_scope->__pyx_v_sentence));
     __Pyx_XDECREF(((PyObject *)__pyx_cur_scope->__pyx_v_sentence));
     __Pyx_GIVEREF(((PyObject *)__pyx_t_4));
     __pyx_cur_scope->__pyx_v_sentence = ((PyObject*)__pyx_t_4);
     __pyx_t_4 = 0;
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":65
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":78
  *         for k in range(hypos.size()):
  *             sentence = GetString(hypos[0][k].words).c_str()
  *             yield sentence.decode('utf8')             # <<<<<<<<<<<<<<
  *         del hypos
  * 
  */
-    __pyx_t_4 = PyObject_GetAttr(((PyObject *)__pyx_cur_scope->__pyx_v_sentence), __pyx_n_s__decode); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = PyObject_GetAttr(((PyObject *)__pyx_cur_scope->__pyx_v_sentence), __pyx_n_s__decode); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
-    __pyx_t_5 = PyObject_Call(__pyx_t_4, ((PyObject *)__pyx_k_tuple_9), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_5 = PyObject_Call(__pyx_t_4, ((PyObject *)__pyx_k_tuple_9), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_5);
     __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
     __pyx_r = __pyx_t_5;
@@ -3533,10 +4156,10 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject
     __pyx_L7_resume_from_yield:;
     __pyx_t_2 = __pyx_cur_scope->__pyx_t_0;
     __pyx_t_3 = __pyx_cur_scope->__pyx_t_1;
-    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":66
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":79
  *             sentence = GetString(hypos[0][k].words).c_str()
  *             yield sentence.decode('utf8')
  *         del hypos             # <<<<<<<<<<<<<<
@@ -3558,13 +4181,13 @@ static PyObject *__pyx_gb_5_cdec_10Hypergraph_18generator4(__pyx_GeneratorObject
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_20intersect(PyObject *__pyx_v_self, PyObject *__pyx_v_lat); /*proto*/
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_20intersect(PyObject *__pyx_v_self, PyObject *__pyx_v_lat) {
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_23intersect(PyObject *__pyx_v_self, PyObject *__pyx_v_lat); /*proto*/
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_23intersect(PyObject *__pyx_v_self, PyObject *__pyx_v_lat) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("intersect (wrapper)", 0);
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_lat), __pyx_ptype_5_cdec_Lattice, 1, "lat", 0))) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_19intersect(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((struct __pyx_obj_5_cdec_Lattice *)__pyx_v_lat));
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_lat), __pyx_ptype_5_cdec_Lattice, 1, "lat", 0))) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 83; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_22intersect(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((struct __pyx_obj_5_cdec_Lattice *)__pyx_v_lat));
   goto __pyx_L0;
   __pyx_L1_error:;
   __pyx_r = NULL;
@@ -3573,7 +4196,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_20intersect(PyObject *__pyx_v_self
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":70
+/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":83
  *     # TODO richer k-best/sample output (feature vectors, trees?)
  * 
  *     def intersect(self, Lattice lat):             # <<<<<<<<<<<<<<
@@ -3581,7 +4204,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_20intersect(PyObject *__pyx_v_self
  * 
  */
 
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_19intersect(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, struct __pyx_obj_5_cdec_Lattice *__pyx_v_lat) {
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_22intersect(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, struct __pyx_obj_5_cdec_Lattice *__pyx_v_lat) {
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
   PyObject *__pyx_t_1 = NULL;
@@ -3590,7 +4213,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_19intersect(struct __pyx_obj_5_cde
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("intersect", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":71
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":84
  * 
  *     def intersect(self, Lattice lat):
  *         return hypergraph.Intersect(lat.lattice[0], self.hg)             # <<<<<<<<<<<<<<
@@ -3598,7 +4221,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_19intersect(struct __pyx_obj_5_cde
  *     def prune(self, beam_alpha=0, density=0, **kwargs):
  */
   __Pyx_XDECREF(__pyx_r);
-  __pyx_t_1 = __Pyx_PyBool_FromLong(HG::Intersect((__pyx_v_lat->lattice[0]), __pyx_v_self->hg)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __Pyx_PyBool_FromLong(HG::Intersect((__pyx_v_lat->lattice[0]), __pyx_v_self->hg)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __pyx_r = __pyx_t_1;
   __pyx_t_1 = 0;
@@ -3617,8 +4240,8 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_19intersect(struct __pyx_obj_5_cde
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_22prune(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_22prune(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_25prune(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_25prune(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
   PyObject *__pyx_v_beam_alpha = 0;
   PyObject *__pyx_v_density = 0;
   PyObject *__pyx_v_kwargs = 0;
@@ -3655,7 +4278,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_22prune(PyObject *__pyx_v_self, Py
         }
       }
       if (unlikely(kw_args > 0)) {
-        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, pos_args, "prune") < 0)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 73; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, pos_args, "prune") < 0)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
       }
     } else {
       switch (PyTuple_GET_SIZE(__pyx_args)) {
@@ -3670,20 +4293,20 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_22prune(PyObject *__pyx_v_self, Py
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L5_argtuple_error:;
-  __Pyx_RaiseArgtupleInvalid("prune", 0, 0, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 73; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __Pyx_RaiseArgtupleInvalid("prune", 0, 0, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[2]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   __pyx_L3_error:;
   __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0;
   __Pyx_AddTraceback("_cdec.Hypergraph.prune", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __Pyx_RefNannyFinishContext();
   return NULL;
   __pyx_L4_argument_unpacking_done:;
-  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_21prune(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), __pyx_v_beam_alpha, __pyx_v_density, __pyx_v_kwargs);
+  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_24prune(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), __pyx_v_beam_alpha, __pyx_v_density, __pyx_v_kwargs);
   __Pyx_XDECREF(__pyx_v_kwargs);
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":73
+/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":86
  *         return hypergraph.Intersect(lat.lattice[0], self.hg)
  * 
  *     def prune(self, beam_alpha=0, density=0, **kwargs):             # <<<<<<<<<<<<<<
@@ -3691,7 +4314,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_22prune(PyObject *__pyx_v_self, Py
  *         if 'csplit_preserve_full_word' in kwargs:
  */
 
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_21prune(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_beam_alpha, PyObject *__pyx_v_density, PyObject *__pyx_v_kwargs) {
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_24prune(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_beam_alpha, PyObject *__pyx_v_density, PyObject *__pyx_v_kwargs) {
   std::vector<bool> *__pyx_v_preserve_mask;
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
@@ -3703,7 +4326,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_21prune(struct __pyx_obj_5_cdec_Hy
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("prune", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":74
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":87
  * 
  *     def prune(self, beam_alpha=0, density=0, **kwargs):
  *         cdef hypergraph.EdgeMask* preserve_mask = NULL             # <<<<<<<<<<<<<<
@@ -3712,17 +4335,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_21prune(struct __pyx_obj_5_cdec_Hy
  */
   __pyx_v_preserve_mask = NULL;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":75
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":88
  *     def prune(self, beam_alpha=0, density=0, **kwargs):
  *         cdef hypergraph.EdgeMask* preserve_mask = NULL
  *         if 'csplit_preserve_full_word' in kwargs:             # <<<<<<<<<<<<<<
  *              preserve_mask = new hypergraph.EdgeMask(self.hg.edges_.size())
  *              preserve_mask[0][hypergraph.GetFullWordEdgeIndex(self.hg[0])] = True
  */
-  __pyx_t_1 = ((PyDict_Contains(((PyObject *)__pyx_v_kwargs), ((PyObject *)__pyx_n_s_10)))); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = ((PyDict_Contains(((PyObject *)__pyx_v_kwargs), ((PyObject *)__pyx_n_s_10)))); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   if (__pyx_t_1) {
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":76
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":89
  *         cdef hypergraph.EdgeMask* preserve_mask = NULL
  *         if 'csplit_preserve_full_word' in kwargs:
  *              preserve_mask = new hypergraph.EdgeMask(self.hg.edges_.size())             # <<<<<<<<<<<<<<
@@ -3731,7 +4354,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_21prune(struct __pyx_obj_5_cdec_Hy
  */
     __pyx_v_preserve_mask = new std::vector<bool>(__pyx_v_self->hg->edges_.size());
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":77
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":90
  *         if 'csplit_preserve_full_word' in kwargs:
  *              preserve_mask = new hypergraph.EdgeMask(self.hg.edges_.size())
  *              preserve_mask[0][hypergraph.GetFullWordEdgeIndex(self.hg[0])] = True             # <<<<<<<<<<<<<<
@@ -3743,15 +4366,15 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_21prune(struct __pyx_obj_5_cdec_Hy
   }
   __pyx_L3:;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":78
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":91
  *              preserve_mask = new hypergraph.EdgeMask(self.hg.edges_.size())
  *              preserve_mask[0][hypergraph.GetFullWordEdgeIndex(self.hg[0])] = True
  *         self.hg.PruneInsideOutside(beam_alpha, density, preserve_mask, False, 1, False)             # <<<<<<<<<<<<<<
  * 
  *     def lattice(self): # TODO direct hg -> lattice conversion in cdec
  */
-  __pyx_t_2 = __pyx_PyFloat_AsDouble(__pyx_v_beam_alpha); if (unlikely((__pyx_t_2 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_t_3 = __pyx_PyFloat_AsDouble(__pyx_v_density); if (unlikely((__pyx_t_3 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __pyx_PyFloat_AsDouble(__pyx_v_beam_alpha); if (unlikely((__pyx_t_2 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = __pyx_PyFloat_AsDouble(__pyx_v_density); if (unlikely((__pyx_t_3 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 91; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_v_self->hg->PruneInsideOutside(__pyx_t_2, __pyx_t_3, __pyx_v_preserve_mask, 0, 1.0, 0);
 
   __pyx_r = Py_None; __Pyx_INCREF(Py_None);
@@ -3766,17 +4389,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_21prune(struct __pyx_obj_5_cdec_Hy
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_24lattice(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_24lattice(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) {
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_27lattice(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_27lattice(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("lattice (wrapper)", 0);
-  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_23lattice(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self));
+  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_26lattice(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self));
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":80
+/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":93
  *         self.hg.PruneInsideOutside(beam_alpha, density, preserve_mask, False, 1, False)
  * 
  *     def lattice(self): # TODO direct hg -> lattice conversion in cdec             # <<<<<<<<<<<<<<
@@ -3784,7 +4407,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_24lattice(PyObject *__pyx_v_self,
  *         return Lattice(eval(plf))
  */
 
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_23lattice(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self) {
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_26lattice(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self) {
   PyObject *__pyx_v_plf = 0;
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
@@ -3796,20 +4419,20 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_23lattice(struct __pyx_obj_5_cdec_
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("lattice", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":81
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":94
  * 
  *     def lattice(self): # TODO direct hg -> lattice conversion in cdec
  *         cdef str plf = hypergraph.AsPLF(self.hg[0], True).c_str()             # <<<<<<<<<<<<<<
  *         return Lattice(eval(plf))
  * 
  */
-  __pyx_t_1 = PyBytes_FromString(HypergraphIO::AsPLF((__pyx_v_self->hg[0]), 1).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 81; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyBytes_FromString(HypergraphIO::AsPLF((__pyx_v_self->hg[0]), 1).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(((PyObject *)__pyx_t_1));
-  if (!(likely(PyString_CheckExact(((PyObject *)__pyx_t_1)))||(PyErr_Format(PyExc_TypeError, "Expected str, got %.200s", Py_TYPE(((PyObject *)__pyx_t_1))->tp_name), 0))) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 81; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (!(likely(PyString_CheckExact(((PyObject *)__pyx_t_1)))||(PyErr_Format(PyExc_TypeError, "Expected str, got %.200s", Py_TYPE(((PyObject *)__pyx_t_1))->tp_name), 0))) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 94; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_v_plf = ((PyObject*)__pyx_t_1);
   __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":82
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":95
  *     def lattice(self): # TODO direct hg -> lattice conversion in cdec
  *         cdef str plf = hypergraph.AsPLF(self.hg[0], True).c_str()
  *         return Lattice(eval(plf))             # <<<<<<<<<<<<<<
@@ -3817,17 +4440,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_23lattice(struct __pyx_obj_5_cdec_
  *     def reweight(self, weights):
  */
   __Pyx_XDECREF(__pyx_r);
-  __pyx_t_1 = __Pyx_Globals(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __Pyx_Globals(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(((PyObject *)__pyx_t_2));
   if (((PyObject *)__pyx_v_plf)) {
-    if (PyDict_SetItem(__pyx_t_2, ((PyObject *)__pyx_n_s__plf), ((PyObject *)__pyx_v_plf)) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    if (PyDict_SetItem(__pyx_t_2, ((PyObject *)__pyx_n_s__plf), ((PyObject *)__pyx_v_plf)) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
   if (((PyObject *)__pyx_v_self)) {
-    if (PyDict_SetItem(__pyx_t_2, ((PyObject *)__pyx_n_s__self), ((PyObject *)__pyx_v_self)) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    if (PyDict_SetItem(__pyx_t_2, ((PyObject *)__pyx_n_s__self), ((PyObject *)__pyx_v_self)) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
-  __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = PyTuple_New(3); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_3);
   __Pyx_INCREF(((PyObject *)__pyx_v_plf));
   PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_v_plf));
@@ -3838,15 +4461,15 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_23lattice(struct __pyx_obj_5_cdec_
   __Pyx_GIVEREF(((PyObject *)__pyx_t_2));
   __pyx_t_1 = 0;
   __pyx_t_2 = 0;
-  __pyx_t_2 = PyObject_Call(__pyx_builtin_eval, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyObject_Call(__pyx_builtin_eval, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
-  __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_3);
   PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2);
   __Pyx_GIVEREF(__pyx_t_2);
   __pyx_t_2 = 0;
-  __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Lattice)), ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Lattice)), ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
   __pyx_r = __pyx_t_2;
@@ -3869,17 +4492,17 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_23lattice(struct __pyx_obj_5_cdec_
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_26reweight(PyObject *__pyx_v_self, PyObject *__pyx_v_weights); /*proto*/
-static PyObject *__pyx_pw_5_cdec_10Hypergraph_26reweight(PyObject *__pyx_v_self, PyObject *__pyx_v_weights) {
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_29reweight(PyObject *__pyx_v_self, PyObject *__pyx_v_weights); /*proto*/
+static PyObject *__pyx_pw_5_cdec_10Hypergraph_29reweight(PyObject *__pyx_v_self, PyObject *__pyx_v_weights) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("reweight (wrapper)", 0);
-  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_25reweight(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((PyObject *)__pyx_v_weights));
+  __pyx_r = __pyx_pf_5_cdec_10Hypergraph_28reweight(((struct __pyx_obj_5_cdec_Hypergraph *)__pyx_v_self), ((PyObject *)__pyx_v_weights));
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":84
+/* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":97
  *         return Lattice(eval(plf))
  * 
  *     def reweight(self, weights):             # <<<<<<<<<<<<<<
@@ -3887,7 +4510,7 @@ static PyObject *__pyx_pw_5_cdec_10Hypergraph_26reweight(PyObject *__pyx_v_self,
  *             self.hg.Reweight((<SparseVector> weights).vector[0])
  */
 
-static PyObject *__pyx_pf_5_cdec_10Hypergraph_25reweight(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_weights) {
+static PyObject *__pyx_pf_5_cdec_10Hypergraph_28reweight(struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_self, PyObject *__pyx_v_weights) {
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
   PyObject *__pyx_t_1 = NULL;
@@ -3898,7 +4521,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_25reweight(struct __pyx_obj_5_cdec
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("reweight", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":85
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":98
  * 
  *     def reweight(self, weights):
  *         if isinstance(weights, SparseVector):             # <<<<<<<<<<<<<<
@@ -3911,7 +4534,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_25reweight(struct __pyx_obj_5_cdec
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
   if (__pyx_t_2) {
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":86
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":99
  *     def reweight(self, weights):
  *         if isinstance(weights, SparseVector):
  *             self.hg.Reweight((<SparseVector> weights).vector[0])             # <<<<<<<<<<<<<<
@@ -3922,7 +4545,7 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_25reweight(struct __pyx_obj_5_cdec
     goto __pyx_L3;
   }
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":87
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":100
  *         if isinstance(weights, SparseVector):
  *             self.hg.Reweight((<SparseVector> weights).vector[0])
  *         elif isinstance(weights, DenseVector):             # <<<<<<<<<<<<<<
@@ -3935,38 +4558,38 @@ static PyObject *__pyx_pf_5_cdec_10Hypergraph_25reweight(struct __pyx_obj_5_cdec
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
   if (__pyx_t_2) {
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":88
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":101
  *             self.hg.Reweight((<SparseVector> weights).vector[0])
  *         elif isinstance(weights, DenseVector):
  *             self.hg.Reweight((<DenseVector> weights).vector[0])             # <<<<<<<<<<<<<<
  *         else:
- *             raise ValueError('cannot reweight hypergraph with %s' % type(weights))
+ *             raise TypeError('cannot reweight hypergraph with %s' % type(weights))
  */
     __pyx_v_self->hg->Reweight((((struct __pyx_obj_5_cdec_DenseVector *)__pyx_v_weights)->vector[0]));
     goto __pyx_L3;
   }
   /*else*/ {
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":90
+    /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":103
  *             self.hg.Reweight((<DenseVector> weights).vector[0])
  *         else:
- *             raise ValueError('cannot reweight hypergraph with %s' % type(weights))             # <<<<<<<<<<<<<<
+ *             raise TypeError('cannot reweight hypergraph with %s' % type(weights))             # <<<<<<<<<<<<<<
  * 
  *     # TODO get feature expectations, get partition function ("inside" score)
  */
-    __pyx_t_1 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_11), ((PyObject *)Py_TYPE(__pyx_v_weights))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_11), ((PyObject *)Py_TYPE(__pyx_v_weights))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(((PyObject *)__pyx_t_1));
-    __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
     PyTuple_SET_ITEM(__pyx_t_3, 0, ((PyObject *)__pyx_t_1));
     __Pyx_GIVEREF(((PyObject *)__pyx_t_1));
     __pyx_t_1 = 0;
-    __pyx_t_1 = PyObject_Call(__pyx_builtin_ValueError, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_1);
     __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
     __Pyx_Raise(__pyx_t_1, 0, 0, 0);
     __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-    {__pyx_filename = __pyx_f[2]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    {__pyx_filename = __pyx_f[2]; __pyx_lineno = 103; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
   __pyx_L3:;
 
@@ -4787,7 +5410,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_8__str__(struct __pyx_obj_5_cdec_Latti
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
-static PyObject *__pyx_gb_5_cdec_7Lattice_12generator5(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
+static PyObject *__pyx_gb_5_cdec_7Lattice_12generator6(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
 
 /* Python wrapper */
 static PyObject *__pyx_pw_5_cdec_7Lattice_11__iter__(PyObject *__pyx_v_self); /*proto*/
@@ -4809,14 +5432,14 @@ static PyObject *__pyx_pw_5_cdec_7Lattice_11__iter__(PyObject *__pyx_v_self) {
  */
 
 static PyObject *__pyx_pf_5_cdec_7Lattice_10__iter__(struct __pyx_obj_5_cdec_Lattice *__pyx_v_self) {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *__pyx_cur_scope;
+  struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *__pyx_cur_scope;
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
   int __pyx_lineno = 0;
   const char *__pyx_filename = NULL;
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__iter__", 0);
-  __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_5___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_5___iter__, __pyx_empty_tuple, NULL);
+  __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_6___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_6___iter__, __pyx_empty_tuple, NULL);
   if (unlikely(!__pyx_cur_scope)) {
     __Pyx_RefNannyFinishContext();
     return NULL;
@@ -4826,7 +5449,7 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_10__iter__(struct __pyx_obj_5_cdec_Lat
   __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
   __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
   {
-    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7Lattice_12generator5, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_7Lattice_12generator6, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_DECREF(__pyx_cur_scope);
     __Pyx_RefNannyFinishContext();
     return (PyObject *) gen;
@@ -4844,9 +5467,9 @@ static PyObject *__pyx_pf_5_cdec_7Lattice_10__iter__(struct __pyx_obj_5_cdec_Lat
   return __pyx_r;
 }
 
-static PyObject *__pyx_gb_5_cdec_7Lattice_12generator5(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
+static PyObject *__pyx_gb_5_cdec_7Lattice_12generator6(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
 {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *)__pyx_generator->closure);
+  struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *)__pyx_generator->closure);
   PyObject *__pyx_r = NULL;
   Py_ssize_t __pyx_t_1;
   unsigned int __pyx_t_2;
@@ -4939,6 +5562,1654 @@ static void __pyx_pf_5_cdec_7Lattice_13__dealloc__(CYTHON_UNUSED struct __pyx_ob
   __Pyx_RefNannyFinishContext();
 }
 
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":3
+ * cimport mteval
+ * 
+ * cdef char* as_str(sentence, error_msg='Cannot convert type %s to str'):             # <<<<<<<<<<<<<<
+ *     cdef bytes ret
+ *     if isinstance(sentence, unicode):
+ */
+
+static char *__pyx_f_5_cdec_as_str(PyObject *__pyx_v_sentence, struct __pyx_opt_args_5_cdec_as_str *__pyx_optional_args) {
+  PyObject *__pyx_v_error_msg = ((PyObject *)__pyx_kp_s_19);
+  PyObject *__pyx_v_ret = 0;
+  char *__pyx_r;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_t_2;
+  PyObject *__pyx_t_3 = NULL;
+  char *__pyx_t_4;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("as_str", 0);
+  if (__pyx_optional_args) {
+    if (__pyx_optional_args->__pyx_n > 0) {
+      __pyx_v_error_msg = __pyx_optional_args->error_msg;
+    }
+  }
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":5
+ * cdef char* as_str(sentence, error_msg='Cannot convert type %s to str'):
+ *     cdef bytes ret
+ *     if isinstance(sentence, unicode):             # <<<<<<<<<<<<<<
+ *         ret = sentence.encode('utf8')
+ *     elif isinstance(sentence, str):
+ */
+  __pyx_t_1 = ((PyObject *)((PyObject*)(&PyUnicode_Type)));
+  __Pyx_INCREF(__pyx_t_1);
+  __pyx_t_2 = __Pyx_TypeCheck(__pyx_v_sentence, __pyx_t_1); 
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  if (__pyx_t_2) {
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":6
+ *     cdef bytes ret
+ *     if isinstance(sentence, unicode):
+ *         ret = sentence.encode('utf8')             # <<<<<<<<<<<<<<
+ *     elif isinstance(sentence, str):
+ *         ret = sentence
+ */
+    __pyx_t_1 = PyObject_GetAttr(__pyx_v_sentence, __pyx_n_s__encode); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_k_tuple_20), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+    if (!(likely(PyBytes_CheckExact(__pyx_t_3))||((__pyx_t_3) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_t_3)->tp_name), 0))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_v_ret = ((PyObject*)__pyx_t_3);
+    __pyx_t_3 = 0;
+    goto __pyx_L3;
+  }
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":7
+ *     if isinstance(sentence, unicode):
+ *         ret = sentence.encode('utf8')
+ *     elif isinstance(sentence, str):             # <<<<<<<<<<<<<<
+ *         ret = sentence
+ *     else:
+ */
+  __pyx_t_3 = ((PyObject *)((PyObject*)(&PyString_Type)));
+  __Pyx_INCREF(__pyx_t_3);
+  __pyx_t_2 = __Pyx_TypeCheck(__pyx_v_sentence, __pyx_t_3); 
+  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+  if (__pyx_t_2) {
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":8
+ *         ret = sentence.encode('utf8')
+ *     elif isinstance(sentence, str):
+ *         ret = sentence             # <<<<<<<<<<<<<<
+ *     else:
+ *         raise TypeError(error_msg % type(sentence))
+ */
+    if (!(likely(PyBytes_CheckExact(__pyx_v_sentence))||((__pyx_v_sentence) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected bytes, got %.200s", Py_TYPE(__pyx_v_sentence)->tp_name), 0))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 8; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_INCREF(__pyx_v_sentence);
+    __pyx_v_ret = ((PyObject*)__pyx_v_sentence);
+    goto __pyx_L3;
+  }
+  /*else*/ {
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":10
+ *         ret = sentence
+ *     else:
+ *         raise TypeError(error_msg % type(sentence))             # <<<<<<<<<<<<<<
+ *     return ret
+ * 
+ */
+    __pyx_t_3 = PyNumber_Remainder(__pyx_v_error_msg, ((PyObject *)Py_TYPE(__pyx_v_sentence))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_3);
+    __Pyx_GIVEREF(__pyx_t_3);
+    __pyx_t_3 = 0;
+    __pyx_t_3 = PyObject_Call(__pyx_builtin_TypeError, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+    __Pyx_Raise(__pyx_t_3, 0, 0, 0);
+    __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
+  __pyx_L3:;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":11
+ *     else:
+ *         raise TypeError(error_msg % type(sentence))
+ *     return ret             # <<<<<<<<<<<<<<
+ * 
+ * cdef class Candidate:
+ */
+  __pyx_t_4 = PyBytes_AsString(((PyObject *)__pyx_v_ret)); if (unlikely((!__pyx_t_4) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 11; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_t_4;
+  goto __pyx_L0;
+
+  __pyx_r = 0;
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_WriteUnraisable("_cdec.as_str", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = 0;
+  __pyx_L0:;
+  __Pyx_XDECREF(__pyx_v_ret);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_9Candidate_5words_1__get__(PyObject *__pyx_v_self); /*proto*/
+static PyObject *__pyx_pw_5_cdec_9Candidate_5words_1__get__(PyObject *__pyx_v_self) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_9Candidate_5words___get__(((struct __pyx_obj_5_cdec_Candidate *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":18
+ * 
+ *     property words:
+ *         def __get__(self):             # <<<<<<<<<<<<<<
+ *             return unicode(GetString(self.candidate.ewords).c_str(), encoding='utf8')
+ * 
+ */
+
+static PyObject *__pyx_pf_5_cdec_9Candidate_5words___get__(struct __pyx_obj_5_cdec_Candidate *__pyx_v_self) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__get__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":19
+ *     property words:
+ *         def __get__(self):
+ *             return unicode(GetString(self.candidate.ewords).c_str(), encoding='utf8')             # <<<<<<<<<<<<<<
+ * 
+ *     property fmap:
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = PyBytes_FromString(TD::GetString(__pyx_v_self->candidate->ewords).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+  __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_t_1));
+  __Pyx_GIVEREF(((PyObject *)__pyx_t_1));
+  __pyx_t_1 = 0;
+  __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+  if (PyDict_SetItem(__pyx_t_1, ((PyObject *)__pyx_n_s__encoding), ((PyObject *)__pyx_n_s__utf8)) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)(&PyUnicode_Type))), ((PyObject *)__pyx_t_2), ((PyObject *)__pyx_t_1)); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_3);
+  __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+  __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+  __pyx_r = __pyx_t_3;
+  __pyx_t_3 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("_cdec.Candidate.words.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_9Candidate_4fmap_1__get__(PyObject *__pyx_v_self); /*proto*/
+static PyObject *__pyx_pw_5_cdec_9Candidate_4fmap_1__get__(PyObject *__pyx_v_self) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_9Candidate_4fmap___get__(((struct __pyx_obj_5_cdec_Candidate *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":22
+ * 
+ *     property fmap:
+ *         def __get__(self):             # <<<<<<<<<<<<<<
+ *             cdef SparseVector fmap = SparseVector()
+ *             fmap.vector = new FastSparseVector[weight_t](self.candidate.fmap)
+ */
+
+static PyObject *__pyx_pf_5_cdec_9Candidate_4fmap___get__(struct __pyx_obj_5_cdec_Candidate *__pyx_v_self) {
+  struct __pyx_obj_5_cdec_SparseVector *__pyx_v_fmap = 0;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__get__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":23
+ *     property fmap:
+ *         def __get__(self):
+ *             cdef SparseVector fmap = SparseVector()             # <<<<<<<<<<<<<<
+ *             fmap.vector = new FastSparseVector[weight_t](self.candidate.fmap)
+ *             return fmap
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SparseVector)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_fmap = ((struct __pyx_obj_5_cdec_SparseVector *)__pyx_t_1);
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":24
+ *         def __get__(self):
+ *             cdef SparseVector fmap = SparseVector()
+ *             fmap.vector = new FastSparseVector[weight_t](self.candidate.fmap)             # <<<<<<<<<<<<<<
+ *             return fmap
+ * 
+ */
+  __pyx_v_fmap->vector = new FastSparseVector<weight_t>(__pyx_v_self->candidate->fmap);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":25
+ *             cdef SparseVector fmap = SparseVector()
+ *             fmap.vector = new FastSparseVector[weight_t](self.candidate.fmap)
+ *             return fmap             # <<<<<<<<<<<<<<
+ * 
+ * cdef class SufficientStats:
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __Pyx_INCREF(((PyObject *)__pyx_v_fmap));
+  __pyx_r = ((PyObject *)__pyx_v_fmap);
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.Candidate.fmap.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XDECREF((PyObject *)__pyx_v_fmap);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_9Candidate_5score_1__get__(PyObject *__pyx_v_self); /*proto*/
+static PyObject *__pyx_pw_5_cdec_9Candidate_5score_1__get__(PyObject *__pyx_v_self) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_9Candidate_5score___get__(((struct __pyx_obj_5_cdec_Candidate *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":15
+ * cdef class Candidate:
+ *     cdef mteval.Candidate* candidate
+ *     cdef public float score             # <<<<<<<<<<<<<<
+ * 
+ *     property words:
+ */
+
+static PyObject *__pyx_pf_5_cdec_9Candidate_5score___get__(struct __pyx_obj_5_cdec_Candidate *__pyx_v_self) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__get__", 0);
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->score); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_r = __pyx_t_1;
+  __pyx_t_1 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.Candidate.score.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static int __pyx_pw_5_cdec_9Candidate_5score_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value); /*proto*/
+static int __pyx_pw_5_cdec_9Candidate_5score_3__set__(PyObject *__pyx_v_self, PyObject *__pyx_v_value) {
+  int __pyx_r;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__set__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_9Candidate_5score_2__set__(((struct __pyx_obj_5_cdec_Candidate *)__pyx_v_self), ((PyObject *)__pyx_v_value));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+static int __pyx_pf_5_cdec_9Candidate_5score_2__set__(struct __pyx_obj_5_cdec_Candidate *__pyx_v_self, PyObject *__pyx_v_value) {
+  int __pyx_r;
+  __Pyx_RefNannyDeclarations
+  float __pyx_t_1;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__set__", 0);
+  __pyx_t_1 = __pyx_PyFloat_AsFloat(__pyx_v_value); if (unlikely((__pyx_t_1 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_v_self->score = __pyx_t_1;
+
+  __pyx_r = 0;
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_AddTraceback("_cdec.Candidate.score.__set__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = -1;
+  __pyx_L0:;
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static void __pyx_pw_5_cdec_15SufficientStats_1__dealloc__(PyObject *__pyx_v_self); /*proto*/
+static void __pyx_pw_5_cdec_15SufficientStats_1__dealloc__(PyObject *__pyx_v_self) {
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0);
+  __pyx_pf_5_cdec_15SufficientStats___dealloc__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":31
+ *     cdef mteval.EvaluationMetric* metric
+ * 
+ *     def __dealloc__(self):             # <<<<<<<<<<<<<<
+ *         del self.stats
+ * 
+ */
+
+static void __pyx_pf_5_cdec_15SufficientStats___dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self) {
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__dealloc__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":32
+ * 
+ *     def __dealloc__(self):
+ *         del self.stats             # <<<<<<<<<<<<<<
+ * 
+ *     property score:
+ */
+  delete __pyx_v_self->stats;
+
+  __Pyx_RefNannyFinishContext();
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_5score_1__get__(PyObject *__pyx_v_self); /*proto*/
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_5score_1__get__(PyObject *__pyx_v_self) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_5score___get__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":35
+ * 
+ *     property score:
+ *         def __get__(self):             # <<<<<<<<<<<<<<
+ *             return self.metric.ComputeScore(self.stats[0])
+ * 
+ */
+
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_5score___get__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__get__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":36
+ *     property score:
+ *         def __get__(self):
+ *             return self.metric.ComputeScore(self.stats[0])             # <<<<<<<<<<<<<<
+ * 
+ *     property detail:
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = PyFloat_FromDouble(__pyx_v_self->metric->ComputeScore((__pyx_v_self->stats[0]))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_r = __pyx_t_1;
+  __pyx_t_1 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.SufficientStats.score.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_6detail_1__get__(PyObject *__pyx_v_self); /*proto*/
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_6detail_1__get__(PyObject *__pyx_v_self) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__get__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_6detail___get__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":39
+ * 
+ *     property detail:
+ *         def __get__(self):             # <<<<<<<<<<<<<<
+ *             return self.metric.DetailedScore(self.stats[0]).c_str()
+ * 
+ */
+
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_6detail___get__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__get__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":40
+ *     property detail:
+ *         def __get__(self):
+ *             return self.metric.DetailedScore(self.stats[0]).c_str()             # <<<<<<<<<<<<<<
+ * 
+ *     def __len__(self):
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = PyBytes_FromString(__pyx_v_self->metric->DetailedScore((__pyx_v_self->stats[0])).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+  __pyx_r = ((PyObject *)__pyx_t_1);
+  __pyx_t_1 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.SufficientStats.detail.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static Py_ssize_t __pyx_pw_5_cdec_15SufficientStats_3__len__(PyObject *__pyx_v_self); /*proto*/
+static Py_ssize_t __pyx_pw_5_cdec_15SufficientStats_3__len__(PyObject *__pyx_v_self) {
+  Py_ssize_t __pyx_r;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__len__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_2__len__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":42
+ *             return self.metric.DetailedScore(self.stats[0]).c_str()
+ * 
+ *     def __len__(self):             # <<<<<<<<<<<<<<
+ *         return self.stats.size()
+ * 
+ */
+
+static Py_ssize_t __pyx_pf_5_cdec_15SufficientStats_2__len__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self) {
+  Py_ssize_t __pyx_r;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__len__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":43
+ * 
+ *     def __len__(self):
+ *         return self.stats.size()             # <<<<<<<<<<<<<<
+ * 
+ *     def __iter__(self):
+ */
+  __pyx_r = __pyx_v_self->stats->size();
+  goto __pyx_L0;
+
+  __pyx_r = 0;
+  __pyx_L0:;
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator7(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_5__iter__(PyObject *__pyx_v_self); /*proto*/
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_5__iter__(PyObject *__pyx_v_self) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__iter__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_4__iter__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":45
+ *         return self.stats.size()
+ * 
+ *     def __iter__(self):             # <<<<<<<<<<<<<<
+ *         for i in range(len(self)):
+ *             yield self.stats[0][i]
+ */
+
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_4__iter__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *__pyx_cur_scope;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__iter__", 0);
+  __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *)__pyx_ptype_5_cdec___pyx_scope_struct_7___iter__->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_7___iter__, __pyx_empty_tuple, NULL);
+  if (unlikely(!__pyx_cur_scope)) {
+    __Pyx_RefNannyFinishContext();
+    return NULL;
+  }
+  __Pyx_GOTREF(__pyx_cur_scope);
+  __pyx_cur_scope->__pyx_v_self = __pyx_v_self;
+  __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
+  __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
+  {
+    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_15SufficientStats_6generator7, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_cur_scope);
+    __Pyx_RefNannyFinishContext();
+    return (PyObject *) gen;
+  }
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_AddTraceback("_cdec.SufficientStats.__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(((PyObject *)__pyx_cur_scope));
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator7(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
+{
+  struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *)__pyx_generator->closure);
+  PyObject *__pyx_r = NULL;
+  Py_ssize_t __pyx_t_1;
+  PyObject *__pyx_t_2 = NULL;
+  PyObject *__pyx_t_3 = NULL;
+  PyObject *(*__pyx_t_4)(PyObject *);
+  unsigned int __pyx_t_5;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("None", 0);
+  switch (__pyx_generator->resume_label) {
+    case 0: goto __pyx_L3_first_run;
+    case 1: goto __pyx_L6_resume_from_yield;
+    default: /* CPython raises the right error here */
+    __Pyx_RefNannyFinishContext();
+    return NULL;
+  }
+  __pyx_L3_first_run:;
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":46
+ * 
+ *     def __iter__(self):
+ *         for i in range(len(self)):             # <<<<<<<<<<<<<<
+ *             yield self.stats[0][i]
+ * 
+ */
+  __pyx_t_1 = PyObject_Length(((PyObject *)__pyx_cur_scope->__pyx_v_self)); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyInt_FromSsize_t(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_3);
+  PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2);
+  __Pyx_GIVEREF(__pyx_t_2);
+  __pyx_t_2 = 0;
+  __pyx_t_2 = PyObject_Call(__pyx_builtin_range, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
+  if (PyList_CheckExact(__pyx_t_2) || PyTuple_CheckExact(__pyx_t_2)) {
+    __pyx_t_3 = __pyx_t_2; __Pyx_INCREF(__pyx_t_3); __pyx_t_1 = 0;
+    __pyx_t_4 = NULL;
+  } else {
+    __pyx_t_1 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __pyx_t_4 = Py_TYPE(__pyx_t_3)->tp_iternext;
+  }
+  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+  for (;;) {
+    if (!__pyx_t_4 && PyList_CheckExact(__pyx_t_3)) {
+      if (__pyx_t_1 >= PyList_GET_SIZE(__pyx_t_3)) break;
+      __pyx_t_2 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_2); __pyx_t_1++;
+    } else if (!__pyx_t_4 && PyTuple_CheckExact(__pyx_t_3)) {
+      if (__pyx_t_1 >= PyTuple_GET_SIZE(__pyx_t_3)) break;
+      __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1); __Pyx_INCREF(__pyx_t_2); __pyx_t_1++;
+    } else {
+      __pyx_t_2 = __pyx_t_4(__pyx_t_3);
+      if (unlikely(!__pyx_t_2)) {
+        if (PyErr_Occurred()) {
+          if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) PyErr_Clear();
+          else {__pyx_filename = __pyx_f[4]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        }
+        break;
+      }
+      __Pyx_GOTREF(__pyx_t_2);
+    }
+    __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_i);
+    __Pyx_XDECREF(__pyx_cur_scope->__pyx_v_i);
+    __Pyx_GIVEREF(__pyx_t_2);
+    __pyx_cur_scope->__pyx_v_i = __pyx_t_2;
+    __pyx_t_2 = 0;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":47
+ *     def __iter__(self):
+ *         for i in range(len(self)):
+ *             yield self.stats[0][i]             # <<<<<<<<<<<<<<
+ * 
+ *     def __iadd__(SufficientStats self, SufficientStats other):
+ */
+    __pyx_t_5 = __Pyx_PyInt_AsUnsignedInt(__pyx_cur_scope->__pyx_v_i); if (unlikely((__pyx_t_5 == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_2 = PyFloat_FromDouble(((__pyx_cur_scope->__pyx_v_self->stats[0])[__pyx_t_5])); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_2);
+    __pyx_r = __pyx_t_2;
+    __pyx_t_2 = 0;
+    __pyx_cur_scope->__pyx_t_0 = __pyx_t_1;
+    __Pyx_XGIVEREF(__pyx_t_3);
+    __pyx_cur_scope->__pyx_t_1 = __pyx_t_3;
+    __pyx_cur_scope->__pyx_t_2 = __pyx_t_4;
+    __Pyx_XGIVEREF(__pyx_r);
+    __Pyx_RefNannyFinishContext();
+    /* return from generator, yielding value */
+    __pyx_generator->resume_label = 1;
+    return __pyx_r;
+    __pyx_L6_resume_from_yield:;
+    __pyx_t_1 = __pyx_cur_scope->__pyx_t_0;
+    __pyx_t_3 = __pyx_cur_scope->__pyx_t_1;
+    __pyx_cur_scope->__pyx_t_1 = 0;
+    __Pyx_XGOTREF(__pyx_t_3);
+    __pyx_t_4 = __pyx_cur_scope->__pyx_t_2;
+    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
+  __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+  PyErr_SetNone(PyExc_StopIteration);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("__iter__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_L0:;
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_generator->resume_label = -1;
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_8__iadd__(PyObject *__pyx_v_self, PyObject *__pyx_v_other); /*proto*/
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_8__iadd__(PyObject *__pyx_v_self, PyObject *__pyx_v_other) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__iadd__ (wrapper)", 0);
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5_cdec_SufficientStats, 1, "other", 0))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_7__iadd__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self), ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_other));
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":49
+ *             yield self.stats[0][i]
+ * 
+ *     def __iadd__(SufficientStats self, SufficientStats other):             # <<<<<<<<<<<<<<
+ *         self.stats[0] += other.stats[0]
+ *         return self
+ */
+
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__iadd__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self, struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_other) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__iadd__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":50
+ * 
+ *     def __iadd__(SufficientStats self, SufficientStats other):
+ *         self.stats[0] += other.stats[0]             # <<<<<<<<<<<<<<
+ *         return self
+ * 
+ */
+  (__pyx_v_self->stats[0]) += (__pyx_v_other->stats[0]);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":51
+ *     def __iadd__(SufficientStats self, SufficientStats other):
+ *         self.stats[0] += other.stats[0]
+ *         return self             # <<<<<<<<<<<<<<
+ * 
+ *     def __add__(SufficientStats x, SufficientStats y):
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __Pyx_INCREF(((PyObject *)__pyx_v_self));
+  __pyx_r = ((PyObject *)__pyx_v_self);
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_10__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /*proto*/
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_10__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__add__ (wrapper)", 0);
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_x), __pyx_ptype_5_cdec_SufficientStats, 1, "x", 0))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_y), __pyx_ptype_5_cdec_SufficientStats, 1, "y", 0))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_9__add__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_x), ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_y));
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":53
+ *         return self
+ * 
+ *     def __add__(SufficientStats x, SufficientStats y):             # <<<<<<<<<<<<<<
+ *         cdef SufficientStats result = SufficientStats()
+ *         result.stats = new mteval.SufficientStats(mteval.add(x.stats[0], y.stats[0]))
+ */
+
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__add__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_x, struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_y) {
+  struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_result = 0;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__add__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":54
+ * 
+ *     def __add__(SufficientStats x, SufficientStats y):
+ *         cdef SufficientStats result = SufficientStats()             # <<<<<<<<<<<<<<
+ *         result.stats = new mteval.SufficientStats(mteval.add(x.stats[0], y.stats[0]))
+ *         result.metric = x.metric
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SufficientStats)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_result = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1);
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":55
+ *     def __add__(SufficientStats x, SufficientStats y):
+ *         cdef SufficientStats result = SufficientStats()
+ *         result.stats = new mteval.SufficientStats(mteval.add(x.stats[0], y.stats[0]))             # <<<<<<<<<<<<<<
+ *         result.metric = x.metric
+ *         return result
+ */
+  __pyx_v_result->stats = new SufficientStats(operator+((__pyx_v_x->stats[0]), (__pyx_v_y->stats[0])));
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":56
+ *         cdef SufficientStats result = SufficientStats()
+ *         result.stats = new mteval.SufficientStats(mteval.add(x.stats[0], y.stats[0]))
+ *         result.metric = x.metric             # <<<<<<<<<<<<<<
+ *         return result
+ * 
+ */
+  __pyx_v_result->metric = __pyx_v_x->metric;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":57
+ *         result.stats = new mteval.SufficientStats(mteval.add(x.stats[0], y.stats[0]))
+ *         result.metric = x.metric
+ *         return result             # <<<<<<<<<<<<<<
+ * 
+ * cdef class SegmentEvaluator:
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __Pyx_INCREF(((PyObject *)__pyx_v_result));
+  __pyx_r = ((PyObject *)__pyx_v_result);
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.SufficientStats.__add__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XDECREF((PyObject *)__pyx_v_result);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static void __pyx_pw_5_cdec_16SegmentEvaluator_1__dealloc__(PyObject *__pyx_v_self); /*proto*/
+static void __pyx_pw_5_cdec_16SegmentEvaluator_1__dealloc__(PyObject *__pyx_v_self) {
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0);
+  __pyx_pf_5_cdec_16SegmentEvaluator___dealloc__(((struct __pyx_obj_5_cdec_SegmentEvaluator *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":63
+ *     cdef mteval.EvaluationMetric* metric
+ * 
+ *     def __dealloc__(self):             # <<<<<<<<<<<<<<
+ *         del self.scorer
+ * 
+ */
+
+static void __pyx_pf_5_cdec_16SegmentEvaluator___dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self) {
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__dealloc__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":64
+ * 
+ *     def __dealloc__(self):
+ *         del self.scorer             # <<<<<<<<<<<<<<
+ * 
+ *     def evaluate(self, sentence):
+ */
+  delete __pyx_v_self->scorer;
+
+  __Pyx_RefNannyFinishContext();
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_16SegmentEvaluator_3evaluate(PyObject *__pyx_v_self, PyObject *__pyx_v_sentence); /*proto*/
+static PyObject *__pyx_pw_5_cdec_16SegmentEvaluator_3evaluate(PyObject *__pyx_v_self, PyObject *__pyx_v_sentence) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("evaluate (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(((struct __pyx_obj_5_cdec_SegmentEvaluator *)__pyx_v_self), ((PyObject *)__pyx_v_sentence));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":66
+ *         del self.scorer
+ * 
+ *     def evaluate(self, sentence):             # <<<<<<<<<<<<<<
+ *         cdef vector[WordID] hyp
+ *         cdef SufficientStats sf = SufficientStats()
+ */
+
+static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self, PyObject *__pyx_v_sentence) {
+  std::vector<WordID> __pyx_v_hyp;
+  struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_sf = 0;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("evaluate", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":68
+ *     def evaluate(self, sentence):
+ *         cdef vector[WordID] hyp
+ *         cdef SufficientStats sf = SufficientStats()             # <<<<<<<<<<<<<<
+ *         sf.metric = self.metric
+ *         sf.stats = new mteval.SufficientStats()
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SufficientStats)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_sf = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1);
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":69
+ *         cdef vector[WordID] hyp
+ *         cdef SufficientStats sf = SufficientStats()
+ *         sf.metric = self.metric             # <<<<<<<<<<<<<<
+ *         sf.stats = new mteval.SufficientStats()
+ *         ConvertSentence(string(as_str(sentence.strip())), &hyp)
+ */
+  __pyx_v_sf->metric = __pyx_v_self->metric;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":70
+ *         cdef SufficientStats sf = SufficientStats()
+ *         sf.metric = self.metric
+ *         sf.stats = new mteval.SufficientStats()             # <<<<<<<<<<<<<<
+ *         ConvertSentence(string(as_str(sentence.strip())), &hyp)
+ *         self.scorer.get().Evaluate(hyp, sf.stats)
+ */
+  __pyx_v_sf->stats = new SufficientStats();
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":71
+ *         sf.metric = self.metric
+ *         sf.stats = new mteval.SufficientStats()
+ *         ConvertSentence(string(as_str(sentence.strip())), &hyp)             # <<<<<<<<<<<<<<
+ *         self.scorer.get().Evaluate(hyp, sf.stats)
+ *         return sf
+ */
+  __pyx_t_1 = PyObject_GetAttr(__pyx_v_sentence, __pyx_n_s__strip); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  TD::ConvertSentence(std::string(__pyx_f_5_cdec_as_str(__pyx_t_2, NULL)), (&__pyx_v_hyp));
+  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":72
+ *         sf.stats = new mteval.SufficientStats()
+ *         ConvertSentence(string(as_str(sentence.strip())), &hyp)
+ *         self.scorer.get().Evaluate(hyp, sf.stats)             # <<<<<<<<<<<<<<
+ *         return sf
+ * 
+ */
+  __pyx_v_self->scorer->get()->Evaluate(__pyx_v_hyp, __pyx_v_sf->stats);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":73
+ *         ConvertSentence(string(as_str(sentence.strip())), &hyp)
+ *         self.scorer.get().Evaluate(hyp, sf.stats)
+ *         return sf             # <<<<<<<<<<<<<<
+ * 
+ *     def candidate_set(self, Hypergraph hypergraph, unsigned k):
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __Pyx_INCREF(((PyObject *)__pyx_v_sf));
+  __pyx_r = ((PyObject *)__pyx_v_sf);
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_AddTraceback("_cdec.SegmentEvaluator.evaluate", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XDECREF((PyObject *)__pyx_v_sf);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+static PyObject *__pyx_gb_5_cdec_16SegmentEvaluator_6generator8(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_16SegmentEvaluator_5candidate_set(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyObject *__pyx_pw_5_cdec_16SegmentEvaluator_5candidate_set(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_hypergraph = 0;
+  unsigned int __pyx_v_k;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__hypergraph,&__pyx_n_s__k,0};
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("candidate_set (wrapper)", 0);
+  {
+    PyObject* values[2] = {0,0};
+    if (unlikely(__pyx_kwds)) {
+      Py_ssize_t kw_args;
+      const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
+      switch (pos_args) {
+        case  2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+        case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+        case  0: break;
+        default: goto __pyx_L5_argtuple_error;
+      }
+      kw_args = PyDict_Size(__pyx_kwds);
+      switch (pos_args) {
+        case  0:
+        values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__hypergraph);
+        if (likely(values[0])) kw_args--;
+        else goto __pyx_L5_argtuple_error;
+        case  1:
+        values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__k);
+        if (likely(values[1])) kw_args--;
+        else {
+          __Pyx_RaiseArgtupleInvalid("candidate_set", 1, 2, 2, 1); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+        }
+      }
+      if (unlikely(kw_args > 0)) {
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "candidate_set") < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    } else if (PyTuple_GET_SIZE(__pyx_args) != 2) {
+      goto __pyx_L5_argtuple_error;
+    } else {
+      values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+      values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+    }
+    __pyx_v_hypergraph = ((struct __pyx_obj_5_cdec_Hypergraph *)values[0]);
+    __pyx_v_k = __Pyx_PyInt_AsUnsignedInt(values[1]); if (unlikely((__pyx_v_k == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("candidate_set", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("_cdec.SegmentEvaluator.candidate_set", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_hypergraph), __pyx_ptype_5_cdec_Hypergraph, 1, "hypergraph", 0))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(((struct __pyx_obj_5_cdec_SegmentEvaluator *)__pyx_v_self), __pyx_v_hypergraph, __pyx_v_k);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":75
+ *         return sf
+ * 
+ *     def candidate_set(self, Hypergraph hypergraph, unsigned k):             # <<<<<<<<<<<<<<
+ *         cdef mteval.CandidateSet* cs = new mteval.CandidateSet()
+ *         cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get())
+ */
+
+static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self, struct __pyx_obj_5_cdec_Hypergraph *__pyx_v_hypergraph, unsigned int __pyx_v_k) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *__pyx_cur_scope;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("candidate_set", 0);
+  __pyx_cur_scope = (struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *)__pyx_ptype_5_cdec___pyx_scope_struct_8_candidate_set->tp_new(__pyx_ptype_5_cdec___pyx_scope_struct_8_candidate_set, __pyx_empty_tuple, NULL);
+  if (unlikely(!__pyx_cur_scope)) {
+    __Pyx_RefNannyFinishContext();
+    return NULL;
+  }
+  __Pyx_GOTREF(__pyx_cur_scope);
+  __pyx_cur_scope->__pyx_v_self = __pyx_v_self;
+  __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
+  __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
+  __pyx_cur_scope->__pyx_v_hypergraph = __pyx_v_hypergraph;
+  __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_hypergraph);
+  __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_hypergraph);
+  __pyx_cur_scope->__pyx_v_k = __pyx_v_k;
+  {
+    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_16SegmentEvaluator_6generator8, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_cur_scope);
+    __Pyx_RefNannyFinishContext();
+    return (PyObject *) gen;
+  }
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_AddTraceback("_cdec.SegmentEvaluator.candidate_set", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_DECREF(((PyObject *)__pyx_cur_scope));
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+static PyObject *__pyx_gb_5_cdec_16SegmentEvaluator_6generator8(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value) /* generator body */
+{
+  struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *__pyx_cur_scope = ((struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *)__pyx_generator->closure);
+  PyObject *__pyx_r = NULL;
+  unsigned int __pyx_t_1;
+  unsigned int __pyx_t_2;
+  PyObject *__pyx_t_3 = NULL;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("None", 0);
+  switch (__pyx_generator->resume_label) {
+    case 0: goto __pyx_L3_first_run;
+    case 1: goto __pyx_L6_resume_from_yield;
+    default: /* CPython raises the right error here */
+    __Pyx_RefNannyFinishContext();
+    return NULL;
+  }
+  __pyx_L3_first_run:;
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":76
+ * 
+ *     def candidate_set(self, Hypergraph hypergraph, unsigned k):
+ *         cdef mteval.CandidateSet* cs = new mteval.CandidateSet()             # <<<<<<<<<<<<<<
+ *         cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get())
+ *         cdef Candidate candidate
+ */
+  __pyx_cur_scope->__pyx_v_cs = new training::CandidateSet();
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":77
+ *     def candidate_set(self, Hypergraph hypergraph, unsigned k):
+ *         cdef mteval.CandidateSet* cs = new mteval.CandidateSet()
+ *         cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get())             # <<<<<<<<<<<<<<
+ *         cdef Candidate candidate
+ *         cdef unsigned i
+ */
+  __pyx_cur_scope->__pyx_v_cs->AddKBestCandidates((__pyx_cur_scope->__pyx_v_hypergraph->hg[0]), __pyx_cur_scope->__pyx_v_k, __pyx_cur_scope->__pyx_v_self->scorer->get());
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":80
+ *         cdef Candidate candidate
+ *         cdef unsigned i
+ *         for i in range(cs.size()):             # <<<<<<<<<<<<<<
+ *             candidate = Candidate()
+ *             candidate.candidate = &cs[0][i]
+ */
+  __pyx_t_1 = __pyx_cur_scope->__pyx_v_cs->size();
+  for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) {
+    __pyx_cur_scope->__pyx_v_i = __pyx_t_2;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":81
+ *         cdef unsigned i
+ *         for i in range(cs.size()):
+ *             candidate = Candidate()             # <<<<<<<<<<<<<<
+ *             candidate.candidate = &cs[0][i]
+ *             candidate.score = self.metric.ComputeScore(cs[0][i].eval_feats)
+ */
+    __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Candidate)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 81; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_3);
+    __Pyx_XGOTREF(((PyObject *)__pyx_cur_scope->__pyx_v_candidate));
+    __Pyx_XDECREF(((PyObject *)__pyx_cur_scope->__pyx_v_candidate));
+    __Pyx_GIVEREF(__pyx_t_3);
+    __pyx_cur_scope->__pyx_v_candidate = ((struct __pyx_obj_5_cdec_Candidate *)__pyx_t_3);
+    __pyx_t_3 = 0;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":82
+ *         for i in range(cs.size()):
+ *             candidate = Candidate()
+ *             candidate.candidate = &cs[0][i]             # <<<<<<<<<<<<<<
+ *             candidate.score = self.metric.ComputeScore(cs[0][i].eval_feats)
+ *             yield candidate
+ */
+    __pyx_cur_scope->__pyx_v_candidate->candidate = (&((__pyx_cur_scope->__pyx_v_cs[0])[__pyx_cur_scope->__pyx_v_i]));
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":83
+ *             candidate = Candidate()
+ *             candidate.candidate = &cs[0][i]
+ *             candidate.score = self.metric.ComputeScore(cs[0][i].eval_feats)             # <<<<<<<<<<<<<<
+ *             yield candidate
+ *         del cs
+ */
+    __pyx_cur_scope->__pyx_v_candidate->score = __pyx_cur_scope->__pyx_v_self->metric->ComputeScore(((__pyx_cur_scope->__pyx_v_cs[0])[__pyx_cur_scope->__pyx_v_i]).eval_feats);
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":84
+ *             candidate.candidate = &cs[0][i]
+ *             candidate.score = self.metric.ComputeScore(cs[0][i].eval_feats)
+ *             yield candidate             # <<<<<<<<<<<<<<
+ *         del cs
+ * 
+ */
+    __Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_v_candidate));
+    __pyx_r = ((PyObject *)__pyx_cur_scope->__pyx_v_candidate);
+    __pyx_cur_scope->__pyx_t_0 = __pyx_t_1;
+    __pyx_cur_scope->__pyx_t_1 = __pyx_t_2;
+    __Pyx_XGIVEREF(__pyx_r);
+    __Pyx_RefNannyFinishContext();
+    /* return from generator, yielding value */
+    __pyx_generator->resume_label = 1;
+    return __pyx_r;
+    __pyx_L6_resume_from_yield:;
+    __pyx_t_1 = __pyx_cur_scope->__pyx_t_0;
+    __pyx_t_2 = __pyx_cur_scope->__pyx_t_1;
+    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":85
+ *             candidate.score = self.metric.ComputeScore(cs[0][i].eval_feats)
+ *             yield candidate
+ *         del cs             # <<<<<<<<<<<<<<
+ * 
+ * cdef class Scorer:
+ */
+  delete __pyx_cur_scope->__pyx_v_cs;
+  PyErr_SetNone(PyExc_StopIteration);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_3);
+  __Pyx_AddTraceback("candidate_set", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_L0:;
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_generator->resume_label = -1;
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+}
+
+/* Python wrapper */
+static int __pyx_pw_5_cdec_6Scorer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static int __pyx_pw_5_cdec_6Scorer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  char *__pyx_v_name;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__name,0};
+  int __pyx_r;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0);
+  {
+    PyObject* values[1] = {0};
+    if (unlikely(__pyx_kwds)) {
+      Py_ssize_t kw_args;
+      const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
+      switch (pos_args) {
+        case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+        case  0: break;
+        default: goto __pyx_L5_argtuple_error;
+      }
+      kw_args = PyDict_Size(__pyx_kwds);
+      switch (pos_args) {
+        case  0:
+        values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__name);
+        if (likely(values[0])) kw_args--;
+        else goto __pyx_L5_argtuple_error;
+      }
+      if (unlikely(kw_args > 0)) {
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
+      goto __pyx_L5_argtuple_error;
+    } else {
+      values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+    }
+    __pyx_v_name = PyBytes_AsString(values[0]); if (unlikely((!__pyx_v_name) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("_cdec.Scorer.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __Pyx_RefNannyFinishContext();
+  return -1;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_r = __pyx_pf_5_cdec_6Scorer___cinit__(((struct __pyx_obj_5_cdec_Scorer *)__pyx_v_self), __pyx_v_name);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":90
+ *     cdef string* name
+ * 
+ *     def __cinit__(self, char* name):             # <<<<<<<<<<<<<<
+ *         self.name = new string(name)
+ * 
+ */
+
+static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, char *__pyx_v_name) {
+  int __pyx_r;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__cinit__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":91
+ * 
+ *     def __cinit__(self, char* name):
+ *         self.name = new string(name)             # <<<<<<<<<<<<<<
+ * 
+ *     def __call__(self, refs):
+ */
+  __pyx_v_self->name = new std::string(__pyx_v_name);
+
+  __pyx_r = 0;
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_6Scorer_3__call__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyObject *__pyx_pw_5_cdec_6Scorer_3__call__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_refs = 0;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__refs,0};
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__call__ (wrapper)", 0);
+  {
+    PyObject* values[1] = {0};
+    if (unlikely(__pyx_kwds)) {
+      Py_ssize_t kw_args;
+      const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
+      switch (pos_args) {
+        case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+        case  0: break;
+        default: goto __pyx_L5_argtuple_error;
+      }
+      kw_args = PyDict_Size(__pyx_kwds);
+      switch (pos_args) {
+        case  0:
+        values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs);
+        if (likely(values[0])) kw_args--;
+        else goto __pyx_L5_argtuple_error;
+      }
+      if (unlikely(kw_args > 0)) {
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__call__") < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
+      goto __pyx_L5_argtuple_error;
+    } else {
+      values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+    }
+    __pyx_v_refs = values[0];
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("__call__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("_cdec.Scorer.__call__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_r = __pyx_pf_5_cdec_6Scorer_2__call__(((struct __pyx_obj_5_cdec_Scorer *)__pyx_v_self), __pyx_v_refs);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":93
+ *         self.name = new string(name)
+ * 
+ *     def __call__(self, refs):             # <<<<<<<<<<<<<<
+ *         cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])
+ *         if isinstance(refs, unicode) or isinstance(refs, str):
+ */
+
+static PyObject *__pyx_pf_5_cdec_6Scorer_2__call__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, PyObject *__pyx_v_refs) {
+  EvaluationMetric *__pyx_v_metric;
+  std::vector<std::vector<WordID> > *__pyx_v_refsv;
+  std::vector<WordID> *__pyx_v_refv;
+  PyObject *__pyx_v_ref = NULL;
+  struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_evaluator = 0;
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_t_2;
+  int __pyx_t_3;
+  int __pyx_t_4;
+  Py_ssize_t __pyx_t_5;
+  PyObject *(*__pyx_t_6)(PyObject *);
+  PyObject *__pyx_t_7 = NULL;
+  PyObject *__pyx_t_8 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__call__", 0);
+  __Pyx_INCREF(__pyx_v_refs);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":94
+ * 
+ *     def __call__(self, refs):
+ *         cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])             # <<<<<<<<<<<<<<
+ *         if isinstance(refs, unicode) or isinstance(refs, str):
+ *             refs = [refs]
+ */
+  __pyx_v_metric = EvaluationMetric::Instance((__pyx_v_self->name[0]));
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":95
+ *     def __call__(self, refs):
+ *         cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])
+ *         if isinstance(refs, unicode) or isinstance(refs, str):             # <<<<<<<<<<<<<<
+ *             refs = [refs]
+ *         cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]()
+ */
+  __pyx_t_1 = ((PyObject *)((PyObject*)(&PyUnicode_Type)));
+  __Pyx_INCREF(__pyx_t_1);
+  __pyx_t_2 = __Pyx_TypeCheck(__pyx_v_refs, __pyx_t_1); 
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  if (!__pyx_t_2) {
+    __pyx_t_1 = ((PyObject *)((PyObject*)(&PyString_Type)));
+    __Pyx_INCREF(__pyx_t_1);
+    __pyx_t_3 = __Pyx_TypeCheck(__pyx_v_refs, __pyx_t_1); 
+    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+    __pyx_t_4 = __pyx_t_3;
+  } else {
+    __pyx_t_4 = __pyx_t_2;
+  }
+  if (__pyx_t_4) {
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":96
+ *         cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])
+ *         if isinstance(refs, unicode) or isinstance(refs, str):
+ *             refs = [refs]             # <<<<<<<<<<<<<<
+ *         cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]()
+ *         cdef vector[WordID]* refv
+ */
+    __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 96; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    __Pyx_INCREF(__pyx_v_refs);
+    PyList_SET_ITEM(__pyx_t_1, 0, __pyx_v_refs);
+    __Pyx_GIVEREF(__pyx_v_refs);
+    __Pyx_DECREF(__pyx_v_refs);
+    __pyx_v_refs = ((PyObject *)__pyx_t_1);
+    __pyx_t_1 = 0;
+    goto __pyx_L3;
+  }
+  __pyx_L3:;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":97
+ *         if isinstance(refs, unicode) or isinstance(refs, str):
+ *             refs = [refs]
+ *         cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]()             # <<<<<<<<<<<<<<
+ *         cdef vector[WordID]* refv
+ *         cdef bytes ref_str
+ */
+  __pyx_v_refsv = new std::vector<std::vector<WordID> >();
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":100
+ *         cdef vector[WordID]* refv
+ *         cdef bytes ref_str
+ *         for ref in refs:             # <<<<<<<<<<<<<<
+ *             refv = new vector[WordID]()
+ *             ConvertSentence(string(as_str(ref.strip())), refv)
+ */
+  if (PyList_CheckExact(__pyx_v_refs) || PyTuple_CheckExact(__pyx_v_refs)) {
+    __pyx_t_1 = __pyx_v_refs; __Pyx_INCREF(__pyx_t_1); __pyx_t_5 = 0;
+    __pyx_t_6 = NULL;
+  } else {
+    __pyx_t_5 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_v_refs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    __pyx_t_6 = Py_TYPE(__pyx_t_1)->tp_iternext;
+  }
+  for (;;) {
+    if (!__pyx_t_6 && PyList_CheckExact(__pyx_t_1)) {
+      if (__pyx_t_5 >= PyList_GET_SIZE(__pyx_t_1)) break;
+      __pyx_t_7 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_7); __pyx_t_5++;
+    } else if (!__pyx_t_6 && PyTuple_CheckExact(__pyx_t_1)) {
+      if (__pyx_t_5 >= PyTuple_GET_SIZE(__pyx_t_1)) break;
+      __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_5); __Pyx_INCREF(__pyx_t_7); __pyx_t_5++;
+    } else {
+      __pyx_t_7 = __pyx_t_6(__pyx_t_1);
+      if (unlikely(!__pyx_t_7)) {
+        if (PyErr_Occurred()) {
+          if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) PyErr_Clear();
+          else {__pyx_filename = __pyx_f[4]; __pyx_lineno = 100; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        }
+        break;
+      }
+      __Pyx_GOTREF(__pyx_t_7);
+    }
+    __Pyx_XDECREF(__pyx_v_ref);
+    __pyx_v_ref = __pyx_t_7;
+    __pyx_t_7 = 0;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":101
+ *         cdef bytes ref_str
+ *         for ref in refs:
+ *             refv = new vector[WordID]()             # <<<<<<<<<<<<<<
+ *             ConvertSentence(string(as_str(ref.strip())), refv)
+ *             refsv.push_back(refv[0])
+ */
+    __pyx_v_refv = new std::vector<WordID>();
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":102
+ *         for ref in refs:
+ *             refv = new vector[WordID]()
+ *             ConvertSentence(string(as_str(ref.strip())), refv)             # <<<<<<<<<<<<<<
+ *             refsv.push_back(refv[0])
+ *             del refv
+ */
+    __pyx_t_7 = PyObject_GetAttr(__pyx_v_ref, __pyx_n_s__strip); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_7);
+    __pyx_t_8 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_8);
+    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
+    TD::ConvertSentence(std::string(__pyx_f_5_cdec_as_str(__pyx_t_8, NULL)), __pyx_v_refv);
+    __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":103
+ *             refv = new vector[WordID]()
+ *             ConvertSentence(string(as_str(ref.strip())), refv)
+ *             refsv.push_back(refv[0])             # <<<<<<<<<<<<<<
+ *             del refv
+ *         cdef unsigned i
+ */
+    __pyx_v_refsv->push_back((__pyx_v_refv[0]));
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":104
+ *             ConvertSentence(string(as_str(ref.strip())), refv)
+ *             refsv.push_back(refv[0])
+ *             del refv             # <<<<<<<<<<<<<<
+ *         cdef unsigned i
+ *         cdef SegmentEvaluator evaluator = SegmentEvaluator()
+ */
+    delete __pyx_v_refv;
+  }
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":106
+ *             del refv
+ *         cdef unsigned i
+ *         cdef SegmentEvaluator evaluator = SegmentEvaluator()             # <<<<<<<<<<<<<<
+ *         evaluator.metric = metric
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SegmentEvaluator)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 106; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_evaluator = ((struct __pyx_obj_5_cdec_SegmentEvaluator *)__pyx_t_1);
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":107
+ *         cdef unsigned i
+ *         cdef SegmentEvaluator evaluator = SegmentEvaluator()
+ *         evaluator.metric = metric             # <<<<<<<<<<<<<<
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+ *         del refsv # in theory should not delete but store in SegmentEvaluator
+ */
+  __pyx_v_evaluator->metric = __pyx_v_metric;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":108
+ *         cdef SegmentEvaluator evaluator = SegmentEvaluator()
+ *         evaluator.metric = metric
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))             # <<<<<<<<<<<<<<
+ *         del refsv # in theory should not delete but store in SegmentEvaluator
+ *         return evaluator
+ */
+  __pyx_v_evaluator->scorer = new boost::shared_ptr<SegmentEvaluator>(__pyx_v_metric->CreateSegmentEvaluator((__pyx_v_refsv[0])));
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":109
+ *         evaluator.metric = metric
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+ *         del refsv # in theory should not delete but store in SegmentEvaluator             # <<<<<<<<<<<<<<
+ *         return evaluator
+ * 
+ */
+  delete __pyx_v_refsv;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":110
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+ *         del refsv # in theory should not delete but store in SegmentEvaluator
+ *         return evaluator             # <<<<<<<<<<<<<<
+ * 
+ *     def __str__(self):
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __Pyx_INCREF(((PyObject *)__pyx_v_evaluator));
+  __pyx_r = ((PyObject *)__pyx_v_evaluator);
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_7);
+  __Pyx_XDECREF(__pyx_t_8);
+  __Pyx_AddTraceback("_cdec.Scorer.__call__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XDECREF(__pyx_v_ref);
+  __Pyx_XDECREF((PyObject *)__pyx_v_evaluator);
+  __Pyx_XDECREF(__pyx_v_refs);
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_6Scorer_5__str__(PyObject *__pyx_v_self); /*proto*/
+static PyObject *__pyx_pw_5_cdec_6Scorer_5__str__(PyObject *__pyx_v_self) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__str__ (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_6Scorer_4__str__(((struct __pyx_obj_5_cdec_Scorer *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":112
+ *         return evaluator
+ * 
+ *     def __str__(self):             # <<<<<<<<<<<<<<
+ *         return self.name.c_str()
+ * 
+ */
+
+static PyObject *__pyx_pf_5_cdec_6Scorer_4__str__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__str__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":113
+ * 
+ *     def __str__(self):
+ *         return self.name.c_str()             # <<<<<<<<<<<<<<
+ * 
+ * BLEU = Scorer('IBM_BLEU')
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = PyBytes_FromString(__pyx_v_self->name->c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 113; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+  __pyx_r = ((PyObject *)__pyx_t_1);
+  __pyx_t_1 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.Scorer.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
 /* Python wrapper */
 static int __pyx_pw_5_cdec_7Decoder_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
 static int __pyx_pw_5_cdec_7Decoder_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
@@ -5385,7 +7656,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_
     }
     /*finally:*/ {
       if (__pyx_t_3) {
-        __pyx_t_7 = PyObject_Call(__pyx_t_3, __pyx_k_tuple_19, NULL);
+        __pyx_t_7 = PyObject_Call(__pyx_t_3, __pyx_k_tuple_21, NULL);
         __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
         if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_7);
@@ -5532,7 +7803,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec
     __pyx_t_1 = PyObject_GetAttr(__pyx_t_3, __pyx_n_s__encode); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_1);
     __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
-    __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_k_tuple_20), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_k_tuple_22), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
     __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
     __pyx_v_inp = __pyx_t_3;
@@ -5611,7 +7882,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec
  *         if grammar:
  *             self.dec.SetSentenceGrammarFromString(string(<char *> grammar))
  */
-    __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_21), ((PyObject *)Py_TYPE(__pyx_v_sentence))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_23), ((PyObject *)Py_TYPE(__pyx_v_sentence))); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(((PyObject *)__pyx_t_3));
     __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_1);
@@ -6064,9 +8335,9 @@ static PyMethodDef __pyx_methods_5_cdec_SparseVector[] = {
 static PyNumberMethods __pyx_tp_as_number_SparseVector = {
   __pyx_pw_5_cdec_12SparseVector_26__add__, /*nb_add*/
   __pyx_pw_5_cdec_12SparseVector_28__sub__, /*nb_subtract*/
-  0, /*nb_multiply*/
+  __pyx_pw_5_cdec_12SparseVector_30__mul__, /*nb_multiply*/
   #if PY_MAJOR_VERSION < 3
-  0, /*nb_divide*/
+  __pyx_pw_5_cdec_12SparseVector_32__div__, /*nb_divide*/
   #endif
   0, /*nb_remainder*/
   0, /*nb_divmod*/
@@ -6241,11 +8512,12 @@ static PyMethodDef __pyx_methods_5_cdec_Hypergraph[] = {
   {__Pyx_NAMESTR("viterbi_features"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_9viterbi_features, METH_NOARGS, __Pyx_DOCSTR(0)},
   {__Pyx_NAMESTR("kbest"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_11kbest, METH_O, __Pyx_DOCSTR(0)},
   {__Pyx_NAMESTR("kbest_tree"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_14kbest_tree, METH_O, __Pyx_DOCSTR(0)},
-  {__Pyx_NAMESTR("sample"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_17sample, METH_O, __Pyx_DOCSTR(0)},
-  {__Pyx_NAMESTR("intersect"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_20intersect, METH_O, __Pyx_DOCSTR(0)},
-  {__Pyx_NAMESTR("prune"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_22prune, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)},
-  {__Pyx_NAMESTR("lattice"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_24lattice, METH_NOARGS, __Pyx_DOCSTR(0)},
-  {__Pyx_NAMESTR("reweight"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_26reweight, METH_O, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("kbest_features"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_17kbest_features, METH_O, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("sample"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_20sample, METH_O, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("intersect"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_23intersect, METH_O, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("prune"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_25prune, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("lattice"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_27lattice, METH_NOARGS, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("reweight"), (PyCFunction)__pyx_pw_5_cdec_10Hypergraph_29reweight, METH_O, __Pyx_DOCSTR(0)},
   {0, 0, 0, 0}
 };
 
@@ -6598,6 +8870,744 @@ static PyTypeObject __pyx_type_5_cdec_Lattice = {
   #endif
 };
 
+static PyObject *__pyx_tp_new_5_cdec_Candidate(PyTypeObject *t, PyObject *a, PyObject *k) {
+  PyObject *o = (*t->tp_alloc)(t, 0);
+  if (!o) return 0;
+  return o;
+}
+
+static void __pyx_tp_dealloc_5_cdec_Candidate(PyObject *o) {
+  (*Py_TYPE(o)->tp_free)(o);
+}
+
+static PyObject *__pyx_getprop_5_cdec_9Candidate_words(PyObject *o, void *x) {
+  return __pyx_pw_5_cdec_9Candidate_5words_1__get__(o);
+}
+
+static PyObject *__pyx_getprop_5_cdec_9Candidate_fmap(PyObject *o, void *x) {
+  return __pyx_pw_5_cdec_9Candidate_4fmap_1__get__(o);
+}
+
+static PyObject *__pyx_getprop_5_cdec_9Candidate_score(PyObject *o, void *x) {
+  return __pyx_pw_5_cdec_9Candidate_5score_1__get__(o);
+}
+
+static int __pyx_setprop_5_cdec_9Candidate_score(PyObject *o, PyObject *v, void *x) {
+  if (v) {
+    return __pyx_pw_5_cdec_9Candidate_5score_3__set__(o, v);
+  }
+  else {
+    PyErr_SetString(PyExc_NotImplementedError, "__del__");
+    return -1;
+  }
+}
+
+static PyMethodDef __pyx_methods_5_cdec_Candidate[] = {
+  {0, 0, 0, 0}
+};
+
+static struct PyGetSetDef __pyx_getsets_5_cdec_Candidate[] = {
+  {(char *)"words", __pyx_getprop_5_cdec_9Candidate_words, 0, 0, 0},
+  {(char *)"fmap", __pyx_getprop_5_cdec_9Candidate_fmap, 0, 0, 0},
+  {(char *)"score", __pyx_getprop_5_cdec_9Candidate_score, __pyx_setprop_5_cdec_9Candidate_score, 0, 0},
+  {0, 0, 0, 0, 0}
+};
+
+static PyNumberMethods __pyx_tp_as_number_Candidate = {
+  0, /*nb_add*/
+  0, /*nb_subtract*/
+  0, /*nb_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_divide*/
+  #endif
+  0, /*nb_remainder*/
+  0, /*nb_divmod*/
+  0, /*nb_power*/
+  0, /*nb_negative*/
+  0, /*nb_positive*/
+  0, /*nb_absolute*/
+  0, /*nb_nonzero*/
+  0, /*nb_invert*/
+  0, /*nb_lshift*/
+  0, /*nb_rshift*/
+  0, /*nb_and*/
+  0, /*nb_xor*/
+  0, /*nb_or*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_coerce*/
+  #endif
+  0, /*nb_int*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_long*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*nb_float*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_oct*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_hex*/
+  #endif
+  0, /*nb_inplace_add*/
+  0, /*nb_inplace_subtract*/
+  0, /*nb_inplace_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_inplace_divide*/
+  #endif
+  0, /*nb_inplace_remainder*/
+  0, /*nb_inplace_power*/
+  0, /*nb_inplace_lshift*/
+  0, /*nb_inplace_rshift*/
+  0, /*nb_inplace_and*/
+  0, /*nb_inplace_xor*/
+  0, /*nb_inplace_or*/
+  0, /*nb_floor_divide*/
+  0, /*nb_true_divide*/
+  0, /*nb_inplace_floor_divide*/
+  0, /*nb_inplace_true_divide*/
+  #if PY_VERSION_HEX >= 0x02050000
+  0, /*nb_index*/
+  #endif
+};
+
+static PySequenceMethods __pyx_tp_as_sequence_Candidate = {
+  0, /*sq_length*/
+  0, /*sq_concat*/
+  0, /*sq_repeat*/
+  0, /*sq_item*/
+  0, /*sq_slice*/
+  0, /*sq_ass_item*/
+  0, /*sq_ass_slice*/
+  0, /*sq_contains*/
+  0, /*sq_inplace_concat*/
+  0, /*sq_inplace_repeat*/
+};
+
+static PyMappingMethods __pyx_tp_as_mapping_Candidate = {
+  0, /*mp_length*/
+  0, /*mp_subscript*/
+  0, /*mp_ass_subscript*/
+};
+
+static PyBufferProcs __pyx_tp_as_buffer_Candidate = {
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getreadbuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getwritebuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getsegcount*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getcharbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_getbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_releasebuffer*/
+  #endif
+};
+
+static PyTypeObject __pyx_type_5_cdec_Candidate = {
+  PyVarObject_HEAD_INIT(0, 0)
+  __Pyx_NAMESTR("_cdec.Candidate"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec_Candidate), /*tp_basicsize*/
+  0, /*tp_itemsize*/
+  __pyx_tp_dealloc_5_cdec_Candidate, /*tp_dealloc*/
+  0, /*tp_print*/
+  0, /*tp_getattr*/
+  0, /*tp_setattr*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*tp_compare*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*tp_repr*/
+  &__pyx_tp_as_number_Candidate, /*tp_as_number*/
+  &__pyx_tp_as_sequence_Candidate, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping_Candidate, /*tp_as_mapping*/
+  0, /*tp_hash*/
+  0, /*tp_call*/
+  0, /*tp_str*/
+  0, /*tp_getattro*/
+  0, /*tp_setattro*/
+  &__pyx_tp_as_buffer_Candidate, /*tp_as_buffer*/
+  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/
+  0, /*tp_doc*/
+  0, /*tp_traverse*/
+  0, /*tp_clear*/
+  0, /*tp_richcompare*/
+  0, /*tp_weaklistoffset*/
+  0, /*tp_iter*/
+  0, /*tp_iternext*/
+  __pyx_methods_5_cdec_Candidate, /*tp_methods*/
+  0, /*tp_members*/
+  __pyx_getsets_5_cdec_Candidate, /*tp_getset*/
+  0, /*tp_base*/
+  0, /*tp_dict*/
+  0, /*tp_descr_get*/
+  0, /*tp_descr_set*/
+  0, /*tp_dictoffset*/
+  0, /*tp_init*/
+  0, /*tp_alloc*/
+  __pyx_tp_new_5_cdec_Candidate, /*tp_new*/
+  0, /*tp_free*/
+  0, /*tp_is_gc*/
+  0, /*tp_bases*/
+  0, /*tp_mro*/
+  0, /*tp_cache*/
+  0, /*tp_subclasses*/
+  0, /*tp_weaklist*/
+  0, /*tp_del*/
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*tp_version_tag*/
+  #endif
+};
+
+static PyObject *__pyx_tp_new_5_cdec_SufficientStats(PyTypeObject *t, PyObject *a, PyObject *k) {
+  PyObject *o = (*t->tp_alloc)(t, 0);
+  if (!o) return 0;
+  return o;
+}
+
+static void __pyx_tp_dealloc_5_cdec_SufficientStats(PyObject *o) {
+  {
+    PyObject *etype, *eval, *etb;
+    PyErr_Fetch(&etype, &eval, &etb);
+    ++Py_REFCNT(o);
+    __pyx_pw_5_cdec_15SufficientStats_1__dealloc__(o);
+    if (PyErr_Occurred()) PyErr_WriteUnraisable(o);
+    --Py_REFCNT(o);
+    PyErr_Restore(etype, eval, etb);
+  }
+  (*Py_TYPE(o)->tp_free)(o);
+}
+
+static PyObject *__pyx_getprop_5_cdec_15SufficientStats_score(PyObject *o, void *x) {
+  return __pyx_pw_5_cdec_15SufficientStats_5score_1__get__(o);
+}
+
+static PyObject *__pyx_getprop_5_cdec_15SufficientStats_detail(PyObject *o, void *x) {
+  return __pyx_pw_5_cdec_15SufficientStats_6detail_1__get__(o);
+}
+
+static PyMethodDef __pyx_methods_5_cdec_SufficientStats[] = {
+  {0, 0, 0, 0}
+};
+
+static struct PyGetSetDef __pyx_getsets_5_cdec_SufficientStats[] = {
+  {(char *)"score", __pyx_getprop_5_cdec_15SufficientStats_score, 0, 0, 0},
+  {(char *)"detail", __pyx_getprop_5_cdec_15SufficientStats_detail, 0, 0, 0},
+  {0, 0, 0, 0, 0}
+};
+
+static PyNumberMethods __pyx_tp_as_number_SufficientStats = {
+  __pyx_pw_5_cdec_15SufficientStats_10__add__, /*nb_add*/
+  0, /*nb_subtract*/
+  0, /*nb_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_divide*/
+  #endif
+  0, /*nb_remainder*/
+  0, /*nb_divmod*/
+  0, /*nb_power*/
+  0, /*nb_negative*/
+  0, /*nb_positive*/
+  0, /*nb_absolute*/
+  0, /*nb_nonzero*/
+  0, /*nb_invert*/
+  0, /*nb_lshift*/
+  0, /*nb_rshift*/
+  0, /*nb_and*/
+  0, /*nb_xor*/
+  0, /*nb_or*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_coerce*/
+  #endif
+  0, /*nb_int*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_long*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*nb_float*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_oct*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_hex*/
+  #endif
+  __pyx_pw_5_cdec_15SufficientStats_8__iadd__, /*nb_inplace_add*/
+  0, /*nb_inplace_subtract*/
+  0, /*nb_inplace_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_inplace_divide*/
+  #endif
+  0, /*nb_inplace_remainder*/
+  0, /*nb_inplace_power*/
+  0, /*nb_inplace_lshift*/
+  0, /*nb_inplace_rshift*/
+  0, /*nb_inplace_and*/
+  0, /*nb_inplace_xor*/
+  0, /*nb_inplace_or*/
+  0, /*nb_floor_divide*/
+  0, /*nb_true_divide*/
+  0, /*nb_inplace_floor_divide*/
+  0, /*nb_inplace_true_divide*/
+  #if PY_VERSION_HEX >= 0x02050000
+  0, /*nb_index*/
+  #endif
+};
+
+static PySequenceMethods __pyx_tp_as_sequence_SufficientStats = {
+  __pyx_pw_5_cdec_15SufficientStats_3__len__, /*sq_length*/
+  0, /*sq_concat*/
+  0, /*sq_repeat*/
+  0, /*sq_item*/
+  0, /*sq_slice*/
+  0, /*sq_ass_item*/
+  0, /*sq_ass_slice*/
+  0, /*sq_contains*/
+  0, /*sq_inplace_concat*/
+  0, /*sq_inplace_repeat*/
+};
+
+static PyMappingMethods __pyx_tp_as_mapping_SufficientStats = {
+  __pyx_pw_5_cdec_15SufficientStats_3__len__, /*mp_length*/
+  0, /*mp_subscript*/
+  0, /*mp_ass_subscript*/
+};
+
+static PyBufferProcs __pyx_tp_as_buffer_SufficientStats = {
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getreadbuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getwritebuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getsegcount*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getcharbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_getbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_releasebuffer*/
+  #endif
+};
+
+static PyTypeObject __pyx_type_5_cdec_SufficientStats = {
+  PyVarObject_HEAD_INIT(0, 0)
+  __Pyx_NAMESTR("_cdec.SufficientStats"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec_SufficientStats), /*tp_basicsize*/
+  0, /*tp_itemsize*/
+  __pyx_tp_dealloc_5_cdec_SufficientStats, /*tp_dealloc*/
+  0, /*tp_print*/
+  0, /*tp_getattr*/
+  0, /*tp_setattr*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*tp_compare*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*tp_repr*/
+  &__pyx_tp_as_number_SufficientStats, /*tp_as_number*/
+  &__pyx_tp_as_sequence_SufficientStats, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping_SufficientStats, /*tp_as_mapping*/
+  0, /*tp_hash*/
+  0, /*tp_call*/
+  0, /*tp_str*/
+  0, /*tp_getattro*/
+  0, /*tp_setattro*/
+  &__pyx_tp_as_buffer_SufficientStats, /*tp_as_buffer*/
+  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/
+  0, /*tp_doc*/
+  0, /*tp_traverse*/
+  0, /*tp_clear*/
+  0, /*tp_richcompare*/
+  0, /*tp_weaklistoffset*/
+  __pyx_pw_5_cdec_15SufficientStats_5__iter__, /*tp_iter*/
+  0, /*tp_iternext*/
+  __pyx_methods_5_cdec_SufficientStats, /*tp_methods*/
+  0, /*tp_members*/
+  __pyx_getsets_5_cdec_SufficientStats, /*tp_getset*/
+  0, /*tp_base*/
+  0, /*tp_dict*/
+  0, /*tp_descr_get*/
+  0, /*tp_descr_set*/
+  0, /*tp_dictoffset*/
+  0, /*tp_init*/
+  0, /*tp_alloc*/
+  __pyx_tp_new_5_cdec_SufficientStats, /*tp_new*/
+  0, /*tp_free*/
+  0, /*tp_is_gc*/
+  0, /*tp_bases*/
+  0, /*tp_mro*/
+  0, /*tp_cache*/
+  0, /*tp_subclasses*/
+  0, /*tp_weaklist*/
+  0, /*tp_del*/
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*tp_version_tag*/
+  #endif
+};
+
+static PyObject *__pyx_tp_new_5_cdec_SegmentEvaluator(PyTypeObject *t, PyObject *a, PyObject *k) {
+  PyObject *o = (*t->tp_alloc)(t, 0);
+  if (!o) return 0;
+  return o;
+}
+
+static void __pyx_tp_dealloc_5_cdec_SegmentEvaluator(PyObject *o) {
+  {
+    PyObject *etype, *eval, *etb;
+    PyErr_Fetch(&etype, &eval, &etb);
+    ++Py_REFCNT(o);
+    __pyx_pw_5_cdec_16SegmentEvaluator_1__dealloc__(o);
+    if (PyErr_Occurred()) PyErr_WriteUnraisable(o);
+    --Py_REFCNT(o);
+    PyErr_Restore(etype, eval, etb);
+  }
+  (*Py_TYPE(o)->tp_free)(o);
+}
+
+static PyMethodDef __pyx_methods_5_cdec_SegmentEvaluator[] = {
+  {__Pyx_NAMESTR("evaluate"), (PyCFunction)__pyx_pw_5_cdec_16SegmentEvaluator_3evaluate, METH_O, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("candidate_set"), (PyCFunction)__pyx_pw_5_cdec_16SegmentEvaluator_5candidate_set, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)},
+  {0, 0, 0, 0}
+};
+
+static PyNumberMethods __pyx_tp_as_number_SegmentEvaluator = {
+  0, /*nb_add*/
+  0, /*nb_subtract*/
+  0, /*nb_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_divide*/
+  #endif
+  0, /*nb_remainder*/
+  0, /*nb_divmod*/
+  0, /*nb_power*/
+  0, /*nb_negative*/
+  0, /*nb_positive*/
+  0, /*nb_absolute*/
+  0, /*nb_nonzero*/
+  0, /*nb_invert*/
+  0, /*nb_lshift*/
+  0, /*nb_rshift*/
+  0, /*nb_and*/
+  0, /*nb_xor*/
+  0, /*nb_or*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_coerce*/
+  #endif
+  0, /*nb_int*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_long*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*nb_float*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_oct*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_hex*/
+  #endif
+  0, /*nb_inplace_add*/
+  0, /*nb_inplace_subtract*/
+  0, /*nb_inplace_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_inplace_divide*/
+  #endif
+  0, /*nb_inplace_remainder*/
+  0, /*nb_inplace_power*/
+  0, /*nb_inplace_lshift*/
+  0, /*nb_inplace_rshift*/
+  0, /*nb_inplace_and*/
+  0, /*nb_inplace_xor*/
+  0, /*nb_inplace_or*/
+  0, /*nb_floor_divide*/
+  0, /*nb_true_divide*/
+  0, /*nb_inplace_floor_divide*/
+  0, /*nb_inplace_true_divide*/
+  #if PY_VERSION_HEX >= 0x02050000
+  0, /*nb_index*/
+  #endif
+};
+
+static PySequenceMethods __pyx_tp_as_sequence_SegmentEvaluator = {
+  0, /*sq_length*/
+  0, /*sq_concat*/
+  0, /*sq_repeat*/
+  0, /*sq_item*/
+  0, /*sq_slice*/
+  0, /*sq_ass_item*/
+  0, /*sq_ass_slice*/
+  0, /*sq_contains*/
+  0, /*sq_inplace_concat*/
+  0, /*sq_inplace_repeat*/
+};
+
+static PyMappingMethods __pyx_tp_as_mapping_SegmentEvaluator = {
+  0, /*mp_length*/
+  0, /*mp_subscript*/
+  0, /*mp_ass_subscript*/
+};
+
+static PyBufferProcs __pyx_tp_as_buffer_SegmentEvaluator = {
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getreadbuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getwritebuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getsegcount*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getcharbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_getbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_releasebuffer*/
+  #endif
+};
+
+static PyTypeObject __pyx_type_5_cdec_SegmentEvaluator = {
+  PyVarObject_HEAD_INIT(0, 0)
+  __Pyx_NAMESTR("_cdec.SegmentEvaluator"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec_SegmentEvaluator), /*tp_basicsize*/
+  0, /*tp_itemsize*/
+  __pyx_tp_dealloc_5_cdec_SegmentEvaluator, /*tp_dealloc*/
+  0, /*tp_print*/
+  0, /*tp_getattr*/
+  0, /*tp_setattr*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*tp_compare*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*tp_repr*/
+  &__pyx_tp_as_number_SegmentEvaluator, /*tp_as_number*/
+  &__pyx_tp_as_sequence_SegmentEvaluator, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping_SegmentEvaluator, /*tp_as_mapping*/
+  0, /*tp_hash*/
+  0, /*tp_call*/
+  0, /*tp_str*/
+  0, /*tp_getattro*/
+  0, /*tp_setattro*/
+  &__pyx_tp_as_buffer_SegmentEvaluator, /*tp_as_buffer*/
+  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/
+  0, /*tp_doc*/
+  0, /*tp_traverse*/
+  0, /*tp_clear*/
+  0, /*tp_richcompare*/
+  0, /*tp_weaklistoffset*/
+  0, /*tp_iter*/
+  0, /*tp_iternext*/
+  __pyx_methods_5_cdec_SegmentEvaluator, /*tp_methods*/
+  0, /*tp_members*/
+  0, /*tp_getset*/
+  0, /*tp_base*/
+  0, /*tp_dict*/
+  0, /*tp_descr_get*/
+  0, /*tp_descr_set*/
+  0, /*tp_dictoffset*/
+  0, /*tp_init*/
+  0, /*tp_alloc*/
+  __pyx_tp_new_5_cdec_SegmentEvaluator, /*tp_new*/
+  0, /*tp_free*/
+  0, /*tp_is_gc*/
+  0, /*tp_bases*/
+  0, /*tp_mro*/
+  0, /*tp_cache*/
+  0, /*tp_subclasses*/
+  0, /*tp_weaklist*/
+  0, /*tp_del*/
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*tp_version_tag*/
+  #endif
+};
+
+static PyObject *__pyx_tp_new_5_cdec_Scorer(PyTypeObject *t, PyObject *a, PyObject *k) {
+  PyObject *o = (*t->tp_alloc)(t, 0);
+  if (!o) return 0;
+  if (__pyx_pw_5_cdec_6Scorer_1__cinit__(o, a, k) < 0) {
+    Py_DECREF(o); o = 0;
+  }
+  return o;
+}
+
+static void __pyx_tp_dealloc_5_cdec_Scorer(PyObject *o) {
+  (*Py_TYPE(o)->tp_free)(o);
+}
+
+static PyMethodDef __pyx_methods_5_cdec_Scorer[] = {
+  {0, 0, 0, 0}
+};
+
+static PyNumberMethods __pyx_tp_as_number_Scorer = {
+  0, /*nb_add*/
+  0, /*nb_subtract*/
+  0, /*nb_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_divide*/
+  #endif
+  0, /*nb_remainder*/
+  0, /*nb_divmod*/
+  0, /*nb_power*/
+  0, /*nb_negative*/
+  0, /*nb_positive*/
+  0, /*nb_absolute*/
+  0, /*nb_nonzero*/
+  0, /*nb_invert*/
+  0, /*nb_lshift*/
+  0, /*nb_rshift*/
+  0, /*nb_and*/
+  0, /*nb_xor*/
+  0, /*nb_or*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_coerce*/
+  #endif
+  0, /*nb_int*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_long*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*nb_float*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_oct*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_hex*/
+  #endif
+  0, /*nb_inplace_add*/
+  0, /*nb_inplace_subtract*/
+  0, /*nb_inplace_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_inplace_divide*/
+  #endif
+  0, /*nb_inplace_remainder*/
+  0, /*nb_inplace_power*/
+  0, /*nb_inplace_lshift*/
+  0, /*nb_inplace_rshift*/
+  0, /*nb_inplace_and*/
+  0, /*nb_inplace_xor*/
+  0, /*nb_inplace_or*/
+  0, /*nb_floor_divide*/
+  0, /*nb_true_divide*/
+  0, /*nb_inplace_floor_divide*/
+  0, /*nb_inplace_true_divide*/
+  #if PY_VERSION_HEX >= 0x02050000
+  0, /*nb_index*/
+  #endif
+};
+
+static PySequenceMethods __pyx_tp_as_sequence_Scorer = {
+  0, /*sq_length*/
+  0, /*sq_concat*/
+  0, /*sq_repeat*/
+  0, /*sq_item*/
+  0, /*sq_slice*/
+  0, /*sq_ass_item*/
+  0, /*sq_ass_slice*/
+  0, /*sq_contains*/
+  0, /*sq_inplace_concat*/
+  0, /*sq_inplace_repeat*/
+};
+
+static PyMappingMethods __pyx_tp_as_mapping_Scorer = {
+  0, /*mp_length*/
+  0, /*mp_subscript*/
+  0, /*mp_ass_subscript*/
+};
+
+static PyBufferProcs __pyx_tp_as_buffer_Scorer = {
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getreadbuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getwritebuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getsegcount*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getcharbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_getbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_releasebuffer*/
+  #endif
+};
+
+static PyTypeObject __pyx_type_5_cdec_Scorer = {
+  PyVarObject_HEAD_INIT(0, 0)
+  __Pyx_NAMESTR("_cdec.Scorer"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec_Scorer), /*tp_basicsize*/
+  0, /*tp_itemsize*/
+  __pyx_tp_dealloc_5_cdec_Scorer, /*tp_dealloc*/
+  0, /*tp_print*/
+  0, /*tp_getattr*/
+  0, /*tp_setattr*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*tp_compare*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*tp_repr*/
+  &__pyx_tp_as_number_Scorer, /*tp_as_number*/
+  &__pyx_tp_as_sequence_Scorer, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping_Scorer, /*tp_as_mapping*/
+  0, /*tp_hash*/
+  __pyx_pw_5_cdec_6Scorer_3__call__, /*tp_call*/
+  __pyx_pw_5_cdec_6Scorer_5__str__, /*tp_str*/
+  0, /*tp_getattro*/
+  0, /*tp_setattro*/
+  &__pyx_tp_as_buffer_Scorer, /*tp_as_buffer*/
+  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/
+  0, /*tp_doc*/
+  0, /*tp_traverse*/
+  0, /*tp_clear*/
+  0, /*tp_richcompare*/
+  0, /*tp_weaklistoffset*/
+  0, /*tp_iter*/
+  0, /*tp_iternext*/
+  __pyx_methods_5_cdec_Scorer, /*tp_methods*/
+  0, /*tp_members*/
+  0, /*tp_getset*/
+  0, /*tp_base*/
+  0, /*tp_dict*/
+  0, /*tp_descr_get*/
+  0, /*tp_descr_set*/
+  0, /*tp_dictoffset*/
+  0, /*tp_init*/
+  0, /*tp_alloc*/
+  __pyx_tp_new_5_cdec_Scorer, /*tp_new*/
+  0, /*tp_free*/
+  0, /*tp_is_gc*/
+  0, /*tp_bases*/
+  0, /*tp_mro*/
+  0, /*tp_cache*/
+  0, /*tp_subclasses*/
+  0, /*tp_weaklist*/
+  0, /*tp_del*/
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*tp_version_tag*/
+  #endif
+};
+
 static PyObject *__pyx_tp_new_5_cdec_Decoder(PyTypeObject *t, PyObject *a, PyObject *k) {
   struct __pyx_obj_5_cdec_Decoder *p;
   PyObject *o = (*t->tp_alloc)(t, 0);
@@ -7625,26 +10635,233 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_3_kbest_tree = {
   #endif
 };
 
-static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_4_sample(PyTypeObject *t, PyObject *a, PyObject *k) {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *p;
+static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_4_kbest_features(PyTypeObject *t, PyObject *a, PyObject *k) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *p;
   PyObject *o = (*t->tp_alloc)(t, 0);
   if (!o) return 0;
-  p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *)o);
+  p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *)o);
+  p->__pyx_v_fmap = 0;
+  p->__pyx_v_self = 0;
+  p->__pyx_v_size = 0;
+  return o;
+}
+
+static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_4_kbest_features(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *)o;
+  Py_XDECREF(((PyObject *)p->__pyx_v_fmap));
+  Py_XDECREF(((PyObject *)p->__pyx_v_self));
+  Py_XDECREF(p->__pyx_v_size);
+  (*Py_TYPE(o)->tp_free)(o);
+}
+
+static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_4_kbest_features(PyObject *o, visitproc v, void *a) {
+  int e;
+  struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *)o;
+  if (p->__pyx_v_fmap) {
+    e = (*v)(((PyObject*)p->__pyx_v_fmap), a); if (e) return e;
+  }
+  if (p->__pyx_v_self) {
+    e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e;
+  }
+  if (p->__pyx_v_size) {
+    e = (*v)(p->__pyx_v_size, a); if (e) return e;
+  }
+  return 0;
+}
+
+static int __pyx_tp_clear_5_cdec___pyx_scope_struct_4_kbest_features(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features *)o;
+  PyObject* tmp;
+  tmp = ((PyObject*)p->__pyx_v_fmap);
+  p->__pyx_v_fmap = ((struct __pyx_obj_5_cdec_SparseVector *)Py_None); Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  tmp = ((PyObject*)p->__pyx_v_self);
+  p->__pyx_v_self = ((struct __pyx_obj_5_cdec_Hypergraph *)Py_None); Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  tmp = ((PyObject*)p->__pyx_v_size);
+  p->__pyx_v_size = Py_None; Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  return 0;
+}
+
+static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_4_kbest_features[] = {
+  {0, 0, 0, 0}
+};
+
+static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_4_kbest_features = {
+  0, /*nb_add*/
+  0, /*nb_subtract*/
+  0, /*nb_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_divide*/
+  #endif
+  0, /*nb_remainder*/
+  0, /*nb_divmod*/
+  0, /*nb_power*/
+  0, /*nb_negative*/
+  0, /*nb_positive*/
+  0, /*nb_absolute*/
+  0, /*nb_nonzero*/
+  0, /*nb_invert*/
+  0, /*nb_lshift*/
+  0, /*nb_rshift*/
+  0, /*nb_and*/
+  0, /*nb_xor*/
+  0, /*nb_or*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_coerce*/
+  #endif
+  0, /*nb_int*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_long*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*nb_float*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_oct*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_hex*/
+  #endif
+  0, /*nb_inplace_add*/
+  0, /*nb_inplace_subtract*/
+  0, /*nb_inplace_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_inplace_divide*/
+  #endif
+  0, /*nb_inplace_remainder*/
+  0, /*nb_inplace_power*/
+  0, /*nb_inplace_lshift*/
+  0, /*nb_inplace_rshift*/
+  0, /*nb_inplace_and*/
+  0, /*nb_inplace_xor*/
+  0, /*nb_inplace_or*/
+  0, /*nb_floor_divide*/
+  0, /*nb_true_divide*/
+  0, /*nb_inplace_floor_divide*/
+  0, /*nb_inplace_true_divide*/
+  #if PY_VERSION_HEX >= 0x02050000
+  0, /*nb_index*/
+  #endif
+};
+
+static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_4_kbest_features = {
+  0, /*sq_length*/
+  0, /*sq_concat*/
+  0, /*sq_repeat*/
+  0, /*sq_item*/
+  0, /*sq_slice*/
+  0, /*sq_ass_item*/
+  0, /*sq_ass_slice*/
+  0, /*sq_contains*/
+  0, /*sq_inplace_concat*/
+  0, /*sq_inplace_repeat*/
+};
+
+static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_4_kbest_features = {
+  0, /*mp_length*/
+  0, /*mp_subscript*/
+  0, /*mp_ass_subscript*/
+};
+
+static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_4_kbest_features = {
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getreadbuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getwritebuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getsegcount*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getcharbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_getbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_releasebuffer*/
+  #endif
+};
+
+static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_4_kbest_features = {
+  PyVarObject_HEAD_INIT(0, 0)
+  __Pyx_NAMESTR("_cdec.__pyx_scope_struct_4_kbest_features"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_4_kbest_features), /*tp_basicsize*/
+  0, /*tp_itemsize*/
+  __pyx_tp_dealloc_5_cdec___pyx_scope_struct_4_kbest_features, /*tp_dealloc*/
+  0, /*tp_print*/
+  0, /*tp_getattr*/
+  0, /*tp_setattr*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*tp_compare*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*tp_repr*/
+  &__pyx_tp_as_number___pyx_scope_struct_4_kbest_features, /*tp_as_number*/
+  &__pyx_tp_as_sequence___pyx_scope_struct_4_kbest_features, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping___pyx_scope_struct_4_kbest_features, /*tp_as_mapping*/
+  0, /*tp_hash*/
+  0, /*tp_call*/
+  0, /*tp_str*/
+  0, /*tp_getattro*/
+  0, /*tp_setattro*/
+  &__pyx_tp_as_buffer___pyx_scope_struct_4_kbest_features, /*tp_as_buffer*/
+  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
+  0, /*tp_doc*/
+  __pyx_tp_traverse_5_cdec___pyx_scope_struct_4_kbest_features, /*tp_traverse*/
+  __pyx_tp_clear_5_cdec___pyx_scope_struct_4_kbest_features, /*tp_clear*/
+  0, /*tp_richcompare*/
+  0, /*tp_weaklistoffset*/
+  0, /*tp_iter*/
+  0, /*tp_iternext*/
+  __pyx_methods_5_cdec___pyx_scope_struct_4_kbest_features, /*tp_methods*/
+  0, /*tp_members*/
+  0, /*tp_getset*/
+  0, /*tp_base*/
+  0, /*tp_dict*/
+  0, /*tp_descr_get*/
+  0, /*tp_descr_set*/
+  0, /*tp_dictoffset*/
+  0, /*tp_init*/
+  0, /*tp_alloc*/
+  __pyx_tp_new_5_cdec___pyx_scope_struct_4_kbest_features, /*tp_new*/
+  0, /*tp_free*/
+  0, /*tp_is_gc*/
+  0, /*tp_bases*/
+  0, /*tp_mro*/
+  0, /*tp_cache*/
+  0, /*tp_subclasses*/
+  0, /*tp_weaklist*/
+  0, /*tp_del*/
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*tp_version_tag*/
+  #endif
+};
+
+static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_5_sample(PyTypeObject *t, PyObject *a, PyObject *k) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *p;
+  PyObject *o = (*t->tp_alloc)(t, 0);
+  if (!o) return 0;
+  p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *)o);
   p->__pyx_v_self = 0;
   p->__pyx_v_sentence = 0;
   return o;
 }
 
-static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_4_sample(PyObject *o) {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *)o;
+static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_5_sample(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *)o;
   Py_XDECREF(((PyObject *)p->__pyx_v_self));
   Py_XDECREF(((PyObject *)p->__pyx_v_sentence));
   (*Py_TYPE(o)->tp_free)(o);
 }
 
-static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_4_sample(PyObject *o, visitproc v, void *a) {
+static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_5_sample(PyObject *o, visitproc v, void *a) {
   int e;
-  struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *)o;
+  struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *)o;
   if (p->__pyx_v_self) {
     e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e;
   }
@@ -7654,8 +10871,8 @@ static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_4_sample(PyObject *o, vis
   return 0;
 }
 
-static int __pyx_tp_clear_5_cdec___pyx_scope_struct_4_sample(PyObject *o) {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample *)o;
+static int __pyx_tp_clear_5_cdec___pyx_scope_struct_5_sample(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample *)o;
   PyObject* tmp;
   tmp = ((PyObject*)p->__pyx_v_self);
   p->__pyx_v_self = ((struct __pyx_obj_5_cdec_Hypergraph *)Py_None); Py_INCREF(Py_None);
@@ -7666,11 +10883,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_4_sample(PyObject *o) {
   return 0;
 }
 
-static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_4_sample[] = {
+static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_5_sample[] = {
   {0, 0, 0, 0}
 };
 
-static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_4_sample = {
+static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_5_sample = {
   0, /*nb_add*/
   0, /*nb_subtract*/
   0, /*nb_multiply*/
@@ -7728,7 +10945,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_4_sample = {
   #endif
 };
 
-static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_4_sample = {
+static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_5_sample = {
   0, /*sq_length*/
   0, /*sq_concat*/
   0, /*sq_repeat*/
@@ -7741,13 +10958,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_4_sample = {
   0, /*sq_inplace_repeat*/
 };
 
-static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_4_sample = {
+static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_5_sample = {
   0, /*mp_length*/
   0, /*mp_subscript*/
   0, /*mp_ass_subscript*/
 };
 
-static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_4_sample = {
+static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_5_sample = {
   #if PY_MAJOR_VERSION < 3
   0, /*bf_getreadbuffer*/
   #endif
@@ -7768,12 +10985,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_4_sample = {
   #endif
 };
 
-static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_4_sample = {
+static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_5_sample = {
   PyVarObject_HEAD_INIT(0, 0)
-  __Pyx_NAMESTR("_cdec.__pyx_scope_struct_4_sample"), /*tp_name*/
-  sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_4_sample), /*tp_basicsize*/
+  __Pyx_NAMESTR("_cdec.__pyx_scope_struct_5_sample"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_5_sample), /*tp_basicsize*/
   0, /*tp_itemsize*/
-  __pyx_tp_dealloc_5_cdec___pyx_scope_struct_4_sample, /*tp_dealloc*/
+  __pyx_tp_dealloc_5_cdec___pyx_scope_struct_5_sample, /*tp_dealloc*/
   0, /*tp_print*/
   0, /*tp_getattr*/
   0, /*tp_setattr*/
@@ -7783,24 +11000,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_4_sample = {
   0, /*reserved*/
   #endif
   0, /*tp_repr*/
-  &__pyx_tp_as_number___pyx_scope_struct_4_sample, /*tp_as_number*/
-  &__pyx_tp_as_sequence___pyx_scope_struct_4_sample, /*tp_as_sequence*/
-  &__pyx_tp_as_mapping___pyx_scope_struct_4_sample, /*tp_as_mapping*/
+  &__pyx_tp_as_number___pyx_scope_struct_5_sample, /*tp_as_number*/
+  &__pyx_tp_as_sequence___pyx_scope_struct_5_sample, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping___pyx_scope_struct_5_sample, /*tp_as_mapping*/
   0, /*tp_hash*/
   0, /*tp_call*/
   0, /*tp_str*/
   0, /*tp_getattro*/
   0, /*tp_setattro*/
-  &__pyx_tp_as_buffer___pyx_scope_struct_4_sample, /*tp_as_buffer*/
+  &__pyx_tp_as_buffer___pyx_scope_struct_5_sample, /*tp_as_buffer*/
   Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
   0, /*tp_doc*/
-  __pyx_tp_traverse_5_cdec___pyx_scope_struct_4_sample, /*tp_traverse*/
-  __pyx_tp_clear_5_cdec___pyx_scope_struct_4_sample, /*tp_clear*/
+  __pyx_tp_traverse_5_cdec___pyx_scope_struct_5_sample, /*tp_traverse*/
+  __pyx_tp_clear_5_cdec___pyx_scope_struct_5_sample, /*tp_clear*/
   0, /*tp_richcompare*/
   0, /*tp_weaklistoffset*/
   0, /*tp_iter*/
   0, /*tp_iternext*/
-  __pyx_methods_5_cdec___pyx_scope_struct_4_sample, /*tp_methods*/
+  __pyx_methods_5_cdec___pyx_scope_struct_5_sample, /*tp_methods*/
   0, /*tp_members*/
   0, /*tp_getset*/
   0, /*tp_base*/
@@ -7810,7 +11027,7 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_4_sample = {
   0, /*tp_dictoffset*/
   0, /*tp_init*/
   0, /*tp_alloc*/
-  __pyx_tp_new_5_cdec___pyx_scope_struct_4_sample, /*tp_new*/
+  __pyx_tp_new_5_cdec___pyx_scope_struct_5_sample, /*tp_new*/
   0, /*tp_free*/
   0, /*tp_is_gc*/
   0, /*tp_bases*/
@@ -7824,32 +11041,32 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_4_sample = {
   #endif
 };
 
-static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_5___iter__(PyTypeObject *t, PyObject *a, PyObject *k) {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *p;
+static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_6___iter__(PyTypeObject *t, PyObject *a, PyObject *k) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *p;
   PyObject *o = (*t->tp_alloc)(t, 0);
   if (!o) return 0;
-  p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *)o);
+  p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *)o);
   p->__pyx_v_self = 0;
   return o;
 }
 
-static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_5___iter__(PyObject *o) {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *)o;
+static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_6___iter__(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *)o;
   Py_XDECREF(((PyObject *)p->__pyx_v_self));
   (*Py_TYPE(o)->tp_free)(o);
 }
 
-static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_5___iter__(PyObject *o, visitproc v, void *a) {
+static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_6___iter__(PyObject *o, visitproc v, void *a) {
   int e;
-  struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *)o;
+  struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *)o;
   if (p->__pyx_v_self) {
     e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e;
   }
   return 0;
 }
 
-static int __pyx_tp_clear_5_cdec___pyx_scope_struct_5___iter__(PyObject *o) {
-  struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__ *)o;
+static int __pyx_tp_clear_5_cdec___pyx_scope_struct_6___iter__(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__ *)o;
   PyObject* tmp;
   tmp = ((PyObject*)p->__pyx_v_self);
   p->__pyx_v_self = ((struct __pyx_obj_5_cdec_Lattice *)Py_None); Py_INCREF(Py_None);
@@ -7857,11 +11074,11 @@ static int __pyx_tp_clear_5_cdec___pyx_scope_struct_5___iter__(PyObject *o) {
   return 0;
 }
 
-static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_5___iter__[] = {
+static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_6___iter__[] = {
   {0, 0, 0, 0}
 };
 
-static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_5___iter__ = {
+static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_6___iter__ = {
   0, /*nb_add*/
   0, /*nb_subtract*/
   0, /*nb_multiply*/
@@ -7919,7 +11136,7 @@ static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_5___iter__ = {
   #endif
 };
 
-static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_5___iter__ = {
+static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_6___iter__ = {
   0, /*sq_length*/
   0, /*sq_concat*/
   0, /*sq_repeat*/
@@ -7932,13 +11149,13 @@ static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_5___iter__ = {
   0, /*sq_inplace_repeat*/
 };
 
-static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_5___iter__ = {
+static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_6___iter__ = {
   0, /*mp_length*/
   0, /*mp_subscript*/
   0, /*mp_ass_subscript*/
 };
 
-static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_5___iter__ = {
+static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_6___iter__ = {
   #if PY_MAJOR_VERSION < 3
   0, /*bf_getreadbuffer*/
   #endif
@@ -7959,12 +11176,12 @@ static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_5___iter__ = {
   #endif
 };
 
-static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_5___iter__ = {
+static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_6___iter__ = {
   PyVarObject_HEAD_INIT(0, 0)
-  __Pyx_NAMESTR("_cdec.__pyx_scope_struct_5___iter__"), /*tp_name*/
-  sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_5___iter__), /*tp_basicsize*/
+  __Pyx_NAMESTR("_cdec.__pyx_scope_struct_6___iter__"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_6___iter__), /*tp_basicsize*/
   0, /*tp_itemsize*/
-  __pyx_tp_dealloc_5_cdec___pyx_scope_struct_5___iter__, /*tp_dealloc*/
+  __pyx_tp_dealloc_5_cdec___pyx_scope_struct_6___iter__, /*tp_dealloc*/
   0, /*tp_print*/
   0, /*tp_getattr*/
   0, /*tp_setattr*/
@@ -7974,24 +11191,24 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_5___iter__ = {
   0, /*reserved*/
   #endif
   0, /*tp_repr*/
-  &__pyx_tp_as_number___pyx_scope_struct_5___iter__, /*tp_as_number*/
-  &__pyx_tp_as_sequence___pyx_scope_struct_5___iter__, /*tp_as_sequence*/
-  &__pyx_tp_as_mapping___pyx_scope_struct_5___iter__, /*tp_as_mapping*/
+  &__pyx_tp_as_number___pyx_scope_struct_6___iter__, /*tp_as_number*/
+  &__pyx_tp_as_sequence___pyx_scope_struct_6___iter__, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping___pyx_scope_struct_6___iter__, /*tp_as_mapping*/
   0, /*tp_hash*/
   0, /*tp_call*/
   0, /*tp_str*/
   0, /*tp_getattro*/
   0, /*tp_setattro*/
-  &__pyx_tp_as_buffer___pyx_scope_struct_5___iter__, /*tp_as_buffer*/
+  &__pyx_tp_as_buffer___pyx_scope_struct_6___iter__, /*tp_as_buffer*/
   Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
   0, /*tp_doc*/
-  __pyx_tp_traverse_5_cdec___pyx_scope_struct_5___iter__, /*tp_traverse*/
-  __pyx_tp_clear_5_cdec___pyx_scope_struct_5___iter__, /*tp_clear*/
+  __pyx_tp_traverse_5_cdec___pyx_scope_struct_6___iter__, /*tp_traverse*/
+  __pyx_tp_clear_5_cdec___pyx_scope_struct_6___iter__, /*tp_clear*/
   0, /*tp_richcompare*/
   0, /*tp_weaklistoffset*/
   0, /*tp_iter*/
   0, /*tp_iternext*/
-  __pyx_methods_5_cdec___pyx_scope_struct_5___iter__, /*tp_methods*/
+  __pyx_methods_5_cdec___pyx_scope_struct_6___iter__, /*tp_methods*/
   0, /*tp_members*/
   0, /*tp_getset*/
   0, /*tp_base*/
@@ -8001,7 +11218,421 @@ static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_5___iter__ = {
   0, /*tp_dictoffset*/
   0, /*tp_init*/
   0, /*tp_alloc*/
-  __pyx_tp_new_5_cdec___pyx_scope_struct_5___iter__, /*tp_new*/
+  __pyx_tp_new_5_cdec___pyx_scope_struct_6___iter__, /*tp_new*/
+  0, /*tp_free*/
+  0, /*tp_is_gc*/
+  0, /*tp_bases*/
+  0, /*tp_mro*/
+  0, /*tp_cache*/
+  0, /*tp_subclasses*/
+  0, /*tp_weaklist*/
+  0, /*tp_del*/
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*tp_version_tag*/
+  #endif
+};
+
+static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_7___iter__(PyTypeObject *t, PyObject *a, PyObject *k) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *p;
+  PyObject *o = (*t->tp_alloc)(t, 0);
+  if (!o) return 0;
+  p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *)o);
+  p->__pyx_v_i = 0;
+  p->__pyx_v_self = 0;
+  p->__pyx_t_1 = 0;
+  return o;
+}
+
+static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_7___iter__(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *)o;
+  Py_XDECREF(p->__pyx_v_i);
+  Py_XDECREF(((PyObject *)p->__pyx_v_self));
+  Py_XDECREF(p->__pyx_t_1);
+  (*Py_TYPE(o)->tp_free)(o);
+}
+
+static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_7___iter__(PyObject *o, visitproc v, void *a) {
+  int e;
+  struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *)o;
+  if (p->__pyx_v_i) {
+    e = (*v)(p->__pyx_v_i, a); if (e) return e;
+  }
+  if (p->__pyx_v_self) {
+    e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e;
+  }
+  if (p->__pyx_t_1) {
+    e = (*v)(p->__pyx_t_1, a); if (e) return e;
+  }
+  return 0;
+}
+
+static int __pyx_tp_clear_5_cdec___pyx_scope_struct_7___iter__(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ *)o;
+  PyObject* tmp;
+  tmp = ((PyObject*)p->__pyx_v_i);
+  p->__pyx_v_i = Py_None; Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  tmp = ((PyObject*)p->__pyx_v_self);
+  p->__pyx_v_self = ((struct __pyx_obj_5_cdec_SufficientStats *)Py_None); Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  tmp = ((PyObject*)p->__pyx_t_1);
+  p->__pyx_t_1 = Py_None; Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  return 0;
+}
+
+static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_7___iter__[] = {
+  {0, 0, 0, 0}
+};
+
+static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_7___iter__ = {
+  0, /*nb_add*/
+  0, /*nb_subtract*/
+  0, /*nb_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_divide*/
+  #endif
+  0, /*nb_remainder*/
+  0, /*nb_divmod*/
+  0, /*nb_power*/
+  0, /*nb_negative*/
+  0, /*nb_positive*/
+  0, /*nb_absolute*/
+  0, /*nb_nonzero*/
+  0, /*nb_invert*/
+  0, /*nb_lshift*/
+  0, /*nb_rshift*/
+  0, /*nb_and*/
+  0, /*nb_xor*/
+  0, /*nb_or*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_coerce*/
+  #endif
+  0, /*nb_int*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_long*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*nb_float*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_oct*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_hex*/
+  #endif
+  0, /*nb_inplace_add*/
+  0, /*nb_inplace_subtract*/
+  0, /*nb_inplace_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_inplace_divide*/
+  #endif
+  0, /*nb_inplace_remainder*/
+  0, /*nb_inplace_power*/
+  0, /*nb_inplace_lshift*/
+  0, /*nb_inplace_rshift*/
+  0, /*nb_inplace_and*/
+  0, /*nb_inplace_xor*/
+  0, /*nb_inplace_or*/
+  0, /*nb_floor_divide*/
+  0, /*nb_true_divide*/
+  0, /*nb_inplace_floor_divide*/
+  0, /*nb_inplace_true_divide*/
+  #if PY_VERSION_HEX >= 0x02050000
+  0, /*nb_index*/
+  #endif
+};
+
+static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_7___iter__ = {
+  0, /*sq_length*/
+  0, /*sq_concat*/
+  0, /*sq_repeat*/
+  0, /*sq_item*/
+  0, /*sq_slice*/
+  0, /*sq_ass_item*/
+  0, /*sq_ass_slice*/
+  0, /*sq_contains*/
+  0, /*sq_inplace_concat*/
+  0, /*sq_inplace_repeat*/
+};
+
+static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_7___iter__ = {
+  0, /*mp_length*/
+  0, /*mp_subscript*/
+  0, /*mp_ass_subscript*/
+};
+
+static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_7___iter__ = {
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getreadbuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getwritebuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getsegcount*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getcharbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_getbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_releasebuffer*/
+  #endif
+};
+
+static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_7___iter__ = {
+  PyVarObject_HEAD_INIT(0, 0)
+  __Pyx_NAMESTR("_cdec.__pyx_scope_struct_7___iter__"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__), /*tp_basicsize*/
+  0, /*tp_itemsize*/
+  __pyx_tp_dealloc_5_cdec___pyx_scope_struct_7___iter__, /*tp_dealloc*/
+  0, /*tp_print*/
+  0, /*tp_getattr*/
+  0, /*tp_setattr*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*tp_compare*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*tp_repr*/
+  &__pyx_tp_as_number___pyx_scope_struct_7___iter__, /*tp_as_number*/
+  &__pyx_tp_as_sequence___pyx_scope_struct_7___iter__, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping___pyx_scope_struct_7___iter__, /*tp_as_mapping*/
+  0, /*tp_hash*/
+  0, /*tp_call*/
+  0, /*tp_str*/
+  0, /*tp_getattro*/
+  0, /*tp_setattro*/
+  &__pyx_tp_as_buffer___pyx_scope_struct_7___iter__, /*tp_as_buffer*/
+  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
+  0, /*tp_doc*/
+  __pyx_tp_traverse_5_cdec___pyx_scope_struct_7___iter__, /*tp_traverse*/
+  __pyx_tp_clear_5_cdec___pyx_scope_struct_7___iter__, /*tp_clear*/
+  0, /*tp_richcompare*/
+  0, /*tp_weaklistoffset*/
+  0, /*tp_iter*/
+  0, /*tp_iternext*/
+  __pyx_methods_5_cdec___pyx_scope_struct_7___iter__, /*tp_methods*/
+  0, /*tp_members*/
+  0, /*tp_getset*/
+  0, /*tp_base*/
+  0, /*tp_dict*/
+  0, /*tp_descr_get*/
+  0, /*tp_descr_set*/
+  0, /*tp_dictoffset*/
+  0, /*tp_init*/
+  0, /*tp_alloc*/
+  __pyx_tp_new_5_cdec___pyx_scope_struct_7___iter__, /*tp_new*/
+  0, /*tp_free*/
+  0, /*tp_is_gc*/
+  0, /*tp_bases*/
+  0, /*tp_mro*/
+  0, /*tp_cache*/
+  0, /*tp_subclasses*/
+  0, /*tp_weaklist*/
+  0, /*tp_del*/
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*tp_version_tag*/
+  #endif
+};
+
+static PyObject *__pyx_tp_new_5_cdec___pyx_scope_struct_8_candidate_set(PyTypeObject *t, PyObject *a, PyObject *k) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *p;
+  PyObject *o = (*t->tp_alloc)(t, 0);
+  if (!o) return 0;
+  p = ((struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *)o);
+  p->__pyx_v_candidate = 0;
+  p->__pyx_v_hypergraph = 0;
+  p->__pyx_v_self = 0;
+  return o;
+}
+
+static void __pyx_tp_dealloc_5_cdec___pyx_scope_struct_8_candidate_set(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *)o;
+  Py_XDECREF(((PyObject *)p->__pyx_v_candidate));
+  Py_XDECREF(((PyObject *)p->__pyx_v_hypergraph));
+  Py_XDECREF(((PyObject *)p->__pyx_v_self));
+  (*Py_TYPE(o)->tp_free)(o);
+}
+
+static int __pyx_tp_traverse_5_cdec___pyx_scope_struct_8_candidate_set(PyObject *o, visitproc v, void *a) {
+  int e;
+  struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *)o;
+  if (p->__pyx_v_candidate) {
+    e = (*v)(((PyObject*)p->__pyx_v_candidate), a); if (e) return e;
+  }
+  if (p->__pyx_v_hypergraph) {
+    e = (*v)(((PyObject*)p->__pyx_v_hypergraph), a); if (e) return e;
+  }
+  if (p->__pyx_v_self) {
+    e = (*v)(((PyObject*)p->__pyx_v_self), a); if (e) return e;
+  }
+  return 0;
+}
+
+static int __pyx_tp_clear_5_cdec___pyx_scope_struct_8_candidate_set(PyObject *o) {
+  struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *p = (struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set *)o;
+  PyObject* tmp;
+  tmp = ((PyObject*)p->__pyx_v_candidate);
+  p->__pyx_v_candidate = ((struct __pyx_obj_5_cdec_Candidate *)Py_None); Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  tmp = ((PyObject*)p->__pyx_v_hypergraph);
+  p->__pyx_v_hypergraph = ((struct __pyx_obj_5_cdec_Hypergraph *)Py_None); Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  tmp = ((PyObject*)p->__pyx_v_self);
+  p->__pyx_v_self = ((struct __pyx_obj_5_cdec_SegmentEvaluator *)Py_None); Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  return 0;
+}
+
+static PyMethodDef __pyx_methods_5_cdec___pyx_scope_struct_8_candidate_set[] = {
+  {0, 0, 0, 0}
+};
+
+static PyNumberMethods __pyx_tp_as_number___pyx_scope_struct_8_candidate_set = {
+  0, /*nb_add*/
+  0, /*nb_subtract*/
+  0, /*nb_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_divide*/
+  #endif
+  0, /*nb_remainder*/
+  0, /*nb_divmod*/
+  0, /*nb_power*/
+  0, /*nb_negative*/
+  0, /*nb_positive*/
+  0, /*nb_absolute*/
+  0, /*nb_nonzero*/
+  0, /*nb_invert*/
+  0, /*nb_lshift*/
+  0, /*nb_rshift*/
+  0, /*nb_and*/
+  0, /*nb_xor*/
+  0, /*nb_or*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_coerce*/
+  #endif
+  0, /*nb_int*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_long*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*nb_float*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_oct*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_hex*/
+  #endif
+  0, /*nb_inplace_add*/
+  0, /*nb_inplace_subtract*/
+  0, /*nb_inplace_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_inplace_divide*/
+  #endif
+  0, /*nb_inplace_remainder*/
+  0, /*nb_inplace_power*/
+  0, /*nb_inplace_lshift*/
+  0, /*nb_inplace_rshift*/
+  0, /*nb_inplace_and*/
+  0, /*nb_inplace_xor*/
+  0, /*nb_inplace_or*/
+  0, /*nb_floor_divide*/
+  0, /*nb_true_divide*/
+  0, /*nb_inplace_floor_divide*/
+  0, /*nb_inplace_true_divide*/
+  #if PY_VERSION_HEX >= 0x02050000
+  0, /*nb_index*/
+  #endif
+};
+
+static PySequenceMethods __pyx_tp_as_sequence___pyx_scope_struct_8_candidate_set = {
+  0, /*sq_length*/
+  0, /*sq_concat*/
+  0, /*sq_repeat*/
+  0, /*sq_item*/
+  0, /*sq_slice*/
+  0, /*sq_ass_item*/
+  0, /*sq_ass_slice*/
+  0, /*sq_contains*/
+  0, /*sq_inplace_concat*/
+  0, /*sq_inplace_repeat*/
+};
+
+static PyMappingMethods __pyx_tp_as_mapping___pyx_scope_struct_8_candidate_set = {
+  0, /*mp_length*/
+  0, /*mp_subscript*/
+  0, /*mp_ass_subscript*/
+};
+
+static PyBufferProcs __pyx_tp_as_buffer___pyx_scope_struct_8_candidate_set = {
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getreadbuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getwritebuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getsegcount*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getcharbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_getbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_releasebuffer*/
+  #endif
+};
+
+static PyTypeObject __pyx_type_5_cdec___pyx_scope_struct_8_candidate_set = {
+  PyVarObject_HEAD_INIT(0, 0)
+  __Pyx_NAMESTR("_cdec.__pyx_scope_struct_8_candidate_set"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec___pyx_scope_struct_8_candidate_set), /*tp_basicsize*/
+  0, /*tp_itemsize*/
+  __pyx_tp_dealloc_5_cdec___pyx_scope_struct_8_candidate_set, /*tp_dealloc*/
+  0, /*tp_print*/
+  0, /*tp_getattr*/
+  0, /*tp_setattr*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*tp_compare*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*tp_repr*/
+  &__pyx_tp_as_number___pyx_scope_struct_8_candidate_set, /*tp_as_number*/
+  &__pyx_tp_as_sequence___pyx_scope_struct_8_candidate_set, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping___pyx_scope_struct_8_candidate_set, /*tp_as_mapping*/
+  0, /*tp_hash*/
+  0, /*tp_call*/
+  0, /*tp_str*/
+  0, /*tp_getattro*/
+  0, /*tp_setattro*/
+  &__pyx_tp_as_buffer___pyx_scope_struct_8_candidate_set, /*tp_as_buffer*/
+  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
+  0, /*tp_doc*/
+  __pyx_tp_traverse_5_cdec___pyx_scope_struct_8_candidate_set, /*tp_traverse*/
+  __pyx_tp_clear_5_cdec___pyx_scope_struct_8_candidate_set, /*tp_clear*/
+  0, /*tp_richcompare*/
+  0, /*tp_weaklistoffset*/
+  0, /*tp_iter*/
+  0, /*tp_iternext*/
+  __pyx_methods_5_cdec___pyx_scope_struct_8_candidate_set, /*tp_methods*/
+  0, /*tp_members*/
+  0, /*tp_getset*/
+  0, /*tp_base*/
+  0, /*tp_dict*/
+  0, /*tp_descr_get*/
+  0, /*tp_descr_set*/
+  0, /*tp_dictoffset*/
+  0, /*tp_init*/
+  0, /*tp_alloc*/
+  __pyx_tp_new_5_cdec___pyx_scope_struct_8_candidate_set, /*tp_new*/
   0, /*tp_free*/
   0, /*tp_is_gc*/
   0, /*tp_bases*/
@@ -8039,15 +11670,18 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_kp_s_11, __pyx_k_11, sizeof(__pyx_k_11), 0, 0, 1, 0},
   {&__pyx_kp_s_13, __pyx_k_13, sizeof(__pyx_k_13), 0, 0, 1, 0},
   {&__pyx_kp_s_14, __pyx_k_14, sizeof(__pyx_k_14), 0, 0, 1, 0},
+  {&__pyx_kp_s_19, __pyx_k_19, sizeof(__pyx_k_19), 0, 0, 1, 0},
   {&__pyx_kp_s_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 1, 0},
-  {&__pyx_kp_s_21, __pyx_k_21, sizeof(__pyx_k_21), 0, 0, 1, 0},
+  {&__pyx_kp_s_23, __pyx_k_23, sizeof(__pyx_k_23), 0, 0, 1, 0},
+  {&__pyx_n_s__BLEU, __pyx_k__BLEU, sizeof(__pyx_k__BLEU), 0, 0, 1, 1},
   {&__pyx_n_s__Exception, __pyx_k__Exception, sizeof(__pyx_k__Exception), 0, 0, 1, 1},
+  {&__pyx_n_s__IBM_BLEU, __pyx_k__IBM_BLEU, sizeof(__pyx_k__IBM_BLEU), 0, 0, 1, 1},
   {&__pyx_n_s__IndexError, __pyx_k__IndexError, sizeof(__pyx_k__IndexError), 0, 0, 1, 1},
   {&__pyx_n_s__KeyError, __pyx_k__KeyError, sizeof(__pyx_k__KeyError), 0, 0, 1, 1},
   {&__pyx_n_s__NotImplemented, __pyx_k__NotImplemented, sizeof(__pyx_k__NotImplemented), 0, 0, 1, 1},
   {&__pyx_n_s__ParseFailed, __pyx_k__ParseFailed, sizeof(__pyx_k__ParseFailed), 0, 0, 1, 1},
+  {&__pyx_n_s__TER, __pyx_k__TER, sizeof(__pyx_k__TER), 0, 0, 1, 1},
   {&__pyx_n_s__TypeError, __pyx_k__TypeError, sizeof(__pyx_k__TypeError), 0, 0, 1, 1},
-  {&__pyx_n_s__ValueError, __pyx_k__ValueError, sizeof(__pyx_k__ValueError), 0, 0, 1, 1},
   {&__pyx_n_s____enter__, __pyx_k____enter__, sizeof(__pyx_k____enter__), 0, 0, 1, 1},
   {&__pyx_n_s____exit__, __pyx_k____exit__, sizeof(__pyx_k____exit__), 0, 0, 1, 1},
   {&__pyx_n_s____main__, __pyx_k____main__, sizeof(__pyx_k____main__), 0, 0, 1, 1},
@@ -8059,13 +11693,18 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_n_s__density, __pyx_k__density, sizeof(__pyx_k__density), 0, 0, 1, 1},
   {&__pyx_n_s__dot, __pyx_k__dot, sizeof(__pyx_k__dot), 0, 0, 1, 1},
   {&__pyx_n_s__encode, __pyx_k__encode, sizeof(__pyx_k__encode), 0, 0, 1, 1},
+  {&__pyx_n_s__encoding, __pyx_k__encoding, sizeof(__pyx_k__encoding), 0, 0, 1, 1},
   {&__pyx_n_s__enumerate, __pyx_k__enumerate, sizeof(__pyx_k__enumerate), 0, 0, 1, 1},
   {&__pyx_n_s__eval, __pyx_k__eval, sizeof(__pyx_k__eval), 0, 0, 1, 1},
   {&__pyx_n_s__grammar, __pyx_k__grammar, sizeof(__pyx_k__grammar), 0, 0, 1, 1},
+  {&__pyx_n_s__hypergraph, __pyx_k__hypergraph, sizeof(__pyx_k__hypergraph), 0, 0, 1, 1},
   {&__pyx_n_s__inp, __pyx_k__inp, sizeof(__pyx_k__inp), 0, 0, 1, 1},
+  {&__pyx_n_s__k, __pyx_k__k, sizeof(__pyx_k__k), 0, 0, 1, 1},
+  {&__pyx_n_s__name, __pyx_k__name, sizeof(__pyx_k__name), 0, 0, 1, 1},
   {&__pyx_n_s__open, __pyx_k__open, sizeof(__pyx_k__open), 0, 0, 1, 1},
   {&__pyx_n_s__plf, __pyx_k__plf, sizeof(__pyx_k__plf), 0, 0, 1, 1},
   {&__pyx_n_s__range, __pyx_k__range, sizeof(__pyx_k__range), 0, 0, 1, 1},
+  {&__pyx_n_s__refs, __pyx_k__refs, sizeof(__pyx_k__refs), 0, 0, 1, 1},
   {&__pyx_n_s__self, __pyx_k__self, sizeof(__pyx_k__self), 0, 0, 1, 1},
   {&__pyx_n_s__sentence, __pyx_k__sentence, sizeof(__pyx_k__sentence), 0, 0, 1, 1},
   {&__pyx_n_s__split, __pyx_k__split, sizeof(__pyx_k__split), 0, 0, 1, 1},
@@ -8074,14 +11713,13 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {0, 0, 0, 0, 0, 0, 0}
 };
 static int __Pyx_InitCachedBuiltins(void) {
-  __pyx_builtin_Exception = __Pyx_GetName(__pyx_b, __pyx_n_s__Exception); if (!__pyx_builtin_Exception) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_builtin_Exception = __Pyx_GetName(__pyx_b, __pyx_n_s__Exception); if (!__pyx_builtin_Exception) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 13; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_KeyError = __Pyx_GetName(__pyx_b, __pyx_n_s__KeyError); if (!__pyx_builtin_KeyError) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_range = __Pyx_GetName(__pyx_b, __pyx_n_s__range); if (!__pyx_builtin_range) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_builtin_ValueError = __Pyx_GetName(__pyx_b, __pyx_n_s__ValueError); if (!__pyx_builtin_ValueError) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_n_s__TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_NotImplemented = __Pyx_GetName(__pyx_b, __pyx_n_s__NotImplemented); if (!__pyx_builtin_NotImplemented) {__pyx_filename = __pyx_f[1]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_builtin_eval = __Pyx_GetName(__pyx_b, __pyx_n_s__eval); if (!__pyx_builtin_eval) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_builtin_eval = __Pyx_GetName(__pyx_b, __pyx_n_s__eval); if (!__pyx_builtin_eval) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_enumerate = __Pyx_GetName(__pyx_b, __pyx_n_s__enumerate); if (!__pyx_builtin_enumerate) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_builtin_TypeError = __Pyx_GetName(__pyx_b, __pyx_n_s__TypeError); if (!__pyx_builtin_TypeError) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_IndexError = __Pyx_GetName(__pyx_b, __pyx_n_s__IndexError); if (!__pyx_builtin_IndexError) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 21; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_open = __Pyx_GetName(__pyx_b, __pyx_n_s__open); if (!__pyx_builtin_open) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   return 0;
@@ -8095,7 +11733,7 @@ static int __Pyx_InitCachedConstants(void) {
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/vectors.pxi":69
  *         elif op == 3: # !=
- *             return not (self == other)
+ *             return not (x == y)
  *         raise NotImplemented('comparison not implemented for SparseVector')             # <<<<<<<<<<<<<<
  * 
  *     def __len__(self):
@@ -8177,14 +11815,14 @@ static int __Pyx_InitCachedConstants(void) {
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8));
   __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_8));
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":65
+  /* "/Users/vchahun/Sandbox/cdec/python/src/hypergraph.pxi":78
  *         for k in range(hypos.size()):
  *             sentence = GetString(hypos[0][k].words).c_str()
  *             yield sentence.decode('utf8')             # <<<<<<<<<<<<<<
  *         del hypos
  * 
  */
-  __pyx_k_tuple_9 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_9)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_k_tuple_9 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_9)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_k_tuple_9);
   __Pyx_INCREF(((PyObject *)__pyx_n_s__utf8));
   PyTuple_SET_ITEM(__pyx_k_tuple_9, 0, ((PyObject *)__pyx_n_s__utf8));
@@ -8261,6 +11899,20 @@ static int __Pyx_InitCachedConstants(void) {
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8));
   __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_18));
 
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":6
+ *     cdef bytes ret
+ *     if isinstance(sentence, unicode):
+ *         ret = sentence.encode('utf8')             # <<<<<<<<<<<<<<
+ *     elif isinstance(sentence, str):
+ *         ret = sentence
+ */
+  __pyx_k_tuple_20 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_20)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_20);
+  __Pyx_INCREF(((PyObject *)__pyx_n_s__utf8));
+  PyTuple_SET_ITEM(__pyx_k_tuple_20, 0, ((PyObject *)__pyx_n_s__utf8));
+  __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_20));
+
   /* "_cdec.pyx":31
  * 
  *     def read_weights(self, cfg):
@@ -8268,18 +11920,18 @@ static int __Pyx_InitCachedConstants(void) {
  *             for line in fp:
  *                 fname, value = line.split()
  */
-  __pyx_k_tuple_19 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_19)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_19);
+  __pyx_k_tuple_21 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_21)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 31; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_21);
   __Pyx_INCREF(Py_None);
-  PyTuple_SET_ITEM(__pyx_k_tuple_19, 0, Py_None);
+  PyTuple_SET_ITEM(__pyx_k_tuple_21, 0, Py_None);
   __Pyx_GIVEREF(Py_None);
   __Pyx_INCREF(Py_None);
-  PyTuple_SET_ITEM(__pyx_k_tuple_19, 1, Py_None);
+  PyTuple_SET_ITEM(__pyx_k_tuple_21, 1, Py_None);
   __Pyx_GIVEREF(Py_None);
   __Pyx_INCREF(Py_None);
-  PyTuple_SET_ITEM(__pyx_k_tuple_19, 2, Py_None);
+  PyTuple_SET_ITEM(__pyx_k_tuple_21, 2, Py_None);
   __Pyx_GIVEREF(Py_None);
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_19));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_21));
 
   /* "_cdec.pyx":38
  *     def translate(self, sentence, grammar=None):
@@ -8288,12 +11940,37 @@ static int __Pyx_InitCachedConstants(void) {
  *         elif isinstance(sentence, str):
  *             inp = sentence.strip()
  */
-  __pyx_k_tuple_20 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_20)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_20);
+  __pyx_k_tuple_22 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_22)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_22);
   __Pyx_INCREF(((PyObject *)__pyx_n_s__utf8));
-  PyTuple_SET_ITEM(__pyx_k_tuple_20, 0, ((PyObject *)__pyx_n_s__utf8));
+  PyTuple_SET_ITEM(__pyx_k_tuple_22, 0, ((PyObject *)__pyx_n_s__utf8));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8));
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_20));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_22));
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":115
+ *         return self.name.c_str()
+ * 
+ * BLEU = Scorer('IBM_BLEU')             # <<<<<<<<<<<<<<
+ * TER = Scorer('TER')
+ */
+  __pyx_k_tuple_24 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_24)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_24);
+  __Pyx_INCREF(((PyObject *)__pyx_n_s__IBM_BLEU));
+  PyTuple_SET_ITEM(__pyx_k_tuple_24, 0, ((PyObject *)__pyx_n_s__IBM_BLEU));
+  __Pyx_GIVEREF(((PyObject *)__pyx_n_s__IBM_BLEU));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_24));
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":116
+ * 
+ * BLEU = Scorer('IBM_BLEU')
+ * TER = Scorer('TER')             # <<<<<<<<<<<<<<
+ */
+  __pyx_k_tuple_25 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_25)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_25);
+  __Pyx_INCREF(((PyObject *)__pyx_n_s__TER));
+  PyTuple_SET_ITEM(__pyx_k_tuple_25, 0, ((PyObject *)__pyx_n_s__TER));
+  __Pyx_GIVEREF(((PyObject *)__pyx_n_s__TER));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_25));
   __Pyx_RefNannyFinishContext();
   return 0;
   __pyx_L1_error:;
@@ -8389,6 +12066,18 @@ PyMODINIT_FUNC PyInit__cdec(void)
   if (PyType_Ready(&__pyx_type_5_cdec_Lattice) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   if (__Pyx_SetAttrString(__pyx_m, "Lattice", (PyObject *)&__pyx_type_5_cdec_Lattice) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec_Lattice = &__pyx_type_5_cdec_Lattice;
+  if (PyType_Ready(&__pyx_type_5_cdec_Candidate) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 13; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__Pyx_SetAttrString(__pyx_m, "Candidate", (PyObject *)&__pyx_type_5_cdec_Candidate) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 13; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec_Candidate = &__pyx_type_5_cdec_Candidate;
+  if (PyType_Ready(&__pyx_type_5_cdec_SufficientStats) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__Pyx_SetAttrString(__pyx_m, "SufficientStats", (PyObject *)&__pyx_type_5_cdec_SufficientStats) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec_SufficientStats = &__pyx_type_5_cdec_SufficientStats;
+  if (PyType_Ready(&__pyx_type_5_cdec_SegmentEvaluator) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__Pyx_SetAttrString(__pyx_m, "SegmentEvaluator", (PyObject *)&__pyx_type_5_cdec_SegmentEvaluator) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec_SegmentEvaluator = &__pyx_type_5_cdec_SegmentEvaluator;
+  if (PyType_Ready(&__pyx_type_5_cdec_Scorer) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 87; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__Pyx_SetAttrString(__pyx_m, "Scorer", (PyObject *)&__pyx_type_5_cdec_Scorer) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 87; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec_Scorer = &__pyx_type_5_cdec_Scorer;
   if (PyType_Ready(&__pyx_type_5_cdec_Decoder) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   if (__Pyx_SetAttrString(__pyx_m, "Decoder", (PyObject *)&__pyx_type_5_cdec_Decoder) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec_Decoder = &__pyx_type_5_cdec_Decoder;
@@ -8400,42 +12089,69 @@ PyMODINIT_FUNC PyInit__cdec(void)
   __pyx_ptype_5_cdec___pyx_scope_struct_2_kbest = &__pyx_type_5_cdec___pyx_scope_struct_2_kbest;
   if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_3_kbest_tree) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec___pyx_scope_struct_3_kbest_tree = &__pyx_type_5_cdec___pyx_scope_struct_3_kbest_tree;
-  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_4_sample) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_ptype_5_cdec___pyx_scope_struct_4_sample = &__pyx_type_5_cdec___pyx_scope_struct_4_sample;
-  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_5___iter__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_ptype_5_cdec___pyx_scope_struct_5___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_5___iter__;
+  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_4_kbest_features) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec___pyx_scope_struct_4_kbest_features = &__pyx_type_5_cdec___pyx_scope_struct_4_kbest_features;
+  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_5_sample) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec___pyx_scope_struct_5_sample = &__pyx_type_5_cdec___pyx_scope_struct_5_sample;
+  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_6___iter__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec___pyx_scope_struct_6___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_6___iter__;
+  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_7___iter__) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec___pyx_scope_struct_7___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_7___iter__;
+  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_8_candidate_set) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 75; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec___pyx_scope_struct_8_candidate_set = &__pyx_type_5_cdec___pyx_scope_struct_8_candidate_set;
   /*--- Type import code ---*/
   /*--- Variable import code ---*/
   /*--- Function import code ---*/
   /*--- Execution code ---*/
 
-  /* "_cdec.pyx":10
- * include "lattice.pxi"
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":115
+ *         return self.name.c_str()
+ * 
+ * BLEU = Scorer('IBM_BLEU')             # <<<<<<<<<<<<<<
+ * TER = Scorer('TER')
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Scorer)), ((PyObject *)__pyx_k_tuple_24), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  if (PyObject_SetAttr(__pyx_m, __pyx_n_s__BLEU, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":116
+ * 
+ * BLEU = Scorer('IBM_BLEU')
+ * TER = Scorer('TER')             # <<<<<<<<<<<<<<
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Scorer)), ((PyObject *)__pyx_k_tuple_25), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  if (PyObject_SetAttr(__pyx_m, __pyx_n_s__TER, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+
+  /* "_cdec.pyx":11
+ * include "mteval.pxi"
  * 
  * SetSilent(True)             # <<<<<<<<<<<<<<
  * 
- * class ParseFailed(Exception):
+ * class ParseFailed(Exception): pass
  */
   SetSilent(1);
 
-  /* "_cdec.pyx":12
+  /* "_cdec.pyx":13
  * SetSilent(True)
  * 
- * class ParseFailed(Exception):             # <<<<<<<<<<<<<<
- *     pass
+ * class ParseFailed(Exception): pass             # <<<<<<<<<<<<<<
  * 
+ * cdef class Decoder:
  */
-  __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 13; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(((PyObject *)__pyx_t_1));
-  __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 13; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_INCREF(__pyx_builtin_Exception);
   PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_builtin_Exception);
   __Pyx_GIVEREF(__pyx_builtin_Exception);
-  __pyx_t_3 = __Pyx_CreateClass(((PyObject *)__pyx_t_2), ((PyObject *)__pyx_t_1), __pyx_n_s__ParseFailed, __pyx_n_s___cdec); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = __Pyx_CreateClass(((PyObject *)__pyx_t_2), ((PyObject *)__pyx_t_1), __pyx_n_s__ParseFailed, __pyx_n_s___cdec); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 13; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_3);
   __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
-  if (PyObject_SetAttr(__pyx_m, __pyx_n_s__ParseFailed, __pyx_t_3) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyObject_SetAttr(__pyx_m, __pyx_n_s__ParseFailed, __pyx_t_3) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 13; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
   __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
 
@@ -8673,6 +12389,18 @@ static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed
     return 0;
 }
 
+static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) {
+    if (unlikely(!type)) {
+        PyErr_Format(PyExc_SystemError, "Missing type object");
+        return 0;
+    }
+    if (likely(PyObject_TypeCheck(obj, type)))
+        return 1;
+    PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s",
+                 Py_TYPE(obj)->tp_name, type->tp_name);
+    return 0;
+}
+
 static void __Pyx_RaiseDoubleKeywordsError(
     const char* func_name,
     PyObject* kw_name)
@@ -8889,18 +12617,6 @@ bad:
     return -1;
 }
 
-static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) {
-    if (unlikely(!type)) {
-        PyErr_Format(PyExc_SystemError, "Missing type object");
-        return 0;
-    }
-    if (likely(PyObject_TypeCheck(obj, type)))
-        return 1;
-    PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s",
-                 Py_TYPE(obj)->tp_name, type->tp_name);
-    return 0;
-}
-
 static CYTHON_INLINE void __Pyx_ExceptionSave(PyObject **type, PyObject **value, PyObject **tb) {
     PyThreadState *tstate = PyThreadState_GET();
     *type = tstate->exc_type;
@@ -9693,6 +13409,25 @@ static int __pyx_Generator_init(void)
     return PyType_Ready(&__pyx_GeneratorType);
 }
 
+static void __Pyx_WriteUnraisable(const char *name, int clineno,
+                                  int lineno, const char *filename) {
+    PyObject *old_exc, *old_val, *old_tb;
+    PyObject *ctx;
+    __Pyx_ErrFetch(&old_exc, &old_val, &old_tb);
+    #if PY_MAJOR_VERSION < 3
+    ctx = PyString_FromString(name);
+    #else
+    ctx = PyUnicode_FromString(name);
+    #endif
+    __Pyx_ErrRestore(old_exc, old_val, old_tb);
+    if (!ctx) {
+        PyErr_WriteUnraisable(Py_None);
+    } else {
+        PyErr_WriteUnraisable(ctx);
+        Py_DECREF(ctx);
+    }
+}
+
 static int __Pyx_check_binary_version(void) {
     char ctversion[4], rtversion[4];
     PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION);
diff --git a/python/src/_cdec.pyx b/python/src/_cdec.pyx
index 698a66f6..6fafbbc0 100644
--- a/python/src/_cdec.pyx
+++ b/python/src/_cdec.pyx
@@ -6,11 +6,11 @@ cimport decoder
 include "vectors.pxi"
 include "hypergraph.pxi"
 include "lattice.pxi"
+include "mteval.pxi"
 
 SetSilent(True)
 
-class ParseFailed(Exception):
-    pass
+class ParseFailed(Exception): pass
 
 cdef class Decoder:
     cdef decoder.Decoder* dec
diff --git a/python/src/hypergraph.pxi b/python/src/hypergraph.pxi
index c226d105..ce78b729 100644
--- a/python/src/hypergraph.pxi
+++ b/python/src/hypergraph.pxi
@@ -53,6 +53,19 @@ cdef class Hypergraph:
             yield tree.decode('utf8')
         del derivations
 
+    def kbest_features(self, size):
+        cdef kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal]* derivations = new kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal](self.hg[0], size)
+        cdef kb.KBestDerivations[FastSparseVector[weight_t], kb.FeatureVectorTraversal].Derivation* derivation
+        cdef SparseVector fmap
+        cdef unsigned k
+        for k in range(size):
+            derivation = derivations.LazyKthBest(self.hg.nodes_.size() - 1, k)
+            if not derivation: break
+            fmap = SparseVector()
+            fmap.vector = new FastSparseVector[weight_t](derivation._yield)
+            yield fmap
+        del derivations
+
     def sample(self, unsigned n):
         cdef vector[hypergraph.Hypothesis]* hypos = new vector[hypergraph.Hypothesis]()
         if self.rng == NULL:
@@ -87,6 +100,6 @@ cdef class Hypergraph:
         elif isinstance(weights, DenseVector):
             self.hg.Reweight((<DenseVector> weights).vector[0])
         else:
-            raise ValueError('cannot reweight hypergraph with %s' % type(weights))
+            raise TypeError('cannot reweight hypergraph with %s' % type(weights))
 
     # TODO get feature expectations, get partition function ("inside" score)
diff --git a/python/src/kbest.pxd b/python/src/kbest.pxd
index f34ac5ed..cef42bd3 100644
--- a/python/src/kbest.pxd
+++ b/python/src/kbest.pxd
@@ -9,6 +9,8 @@ cdef extern from "decoder/viterbi.h":
         pass
     cdef cppclass FTreeTraversal:
         pass
+    cdef cppclass FeatureVectorTraversal:
+        pass
 
 cdef extern from "decoder/kbest.h" namespace "KBest":
     cdef cppclass KBestDerivations[T, Traversal]:
diff --git a/python/src/mteval.pxd b/python/src/mteval.pxd
new file mode 100644
index 00000000..52af6297
--- /dev/null
+++ b/python/src/mteval.pxd
@@ -0,0 +1,47 @@
+from libcpp.vector cimport vector
+from libcpp.string cimport string
+from utils cimport *
+from hypergraph cimport Hypergraph
+
+cdef extern from "mteval/ns.h":
+    cdef cppclass SufficientStats:
+        SufficientStats()
+        SufficientStats(SufficientStats&)
+        unsigned size()
+        float operator[](unsigned i)
+        void swap(SufficientStats& other)
+
+    SufficientStats add "operator+" (SufficientStats&, SufficientStats&)
+
+    cdef cppclass SegmentEvaluator:
+        void Evaluate(vector[WordID]& hyp, SufficientStats* out)
+
+    cdef cppclass EvaluationMetric:
+        string& MetricId()
+        bint IsErrorMetric()
+        float ComputeScore(SufficientStats& stats)
+        string DetailedScore(SufficientStats& stats)
+        shared_ptr[SegmentEvaluator] CreateSegmentEvaluator(vector[vector[WordID]]& refs)
+        ComputeSufficientStatistics(vector[WordID]& hyp,
+                                    vector[WordID]& refs,
+                                    SufficientStats* out)
+
+cdef extern from "mteval/ns.h" namespace "EvaluationMetric":
+    EvaluationMetric* Instance(string& metric_id)
+    EvaluationMetric* Instance() # IBM_BLEU
+
+cdef extern from "training/candidate_set.h" namespace "training":
+    cdef cppclass Candidate "const training::Candidate":
+        vector[WordID] ewords
+        FastSparseVector[weight_t] fmap
+        SufficientStats eval_feats
+
+    cdef cppclass CandidateSet:
+        CandidateSet()
+        unsigned size()
+        Candidate& operator[](unsigned i)
+        void ReadFromFile(string& file)
+        void WriteToFile(string& file)
+        void AddKBestCandidates(Hypergraph& hg,
+                unsigned kbest_size,
+                SegmentEvaluator* scorer)
diff --git a/python/src/mteval.pxi b/python/src/mteval.pxi
new file mode 100644
index 00000000..dabdf927
--- /dev/null
+++ b/python/src/mteval.pxi
@@ -0,0 +1,116 @@
+cimport mteval
+
+cdef char* as_str(sentence, error_msg='Cannot convert type %s to str'):
+    cdef bytes ret
+    if isinstance(sentence, unicode):
+        ret = sentence.encode('utf8')
+    elif isinstance(sentence, str):
+        ret = sentence
+    else:
+        raise TypeError(error_msg % type(sentence))
+    return ret
+
+cdef class Candidate:
+    cdef mteval.Candidate* candidate
+    cdef public float score
+
+    property words:
+        def __get__(self):
+            return unicode(GetString(self.candidate.ewords).c_str(), encoding='utf8')
+
+    property fmap:
+        def __get__(self):
+            cdef SparseVector fmap = SparseVector()
+            fmap.vector = new FastSparseVector[weight_t](self.candidate.fmap)
+            return fmap
+
+cdef class SufficientStats:
+    cdef mteval.SufficientStats* stats
+    cdef mteval.EvaluationMetric* metric
+
+    def __dealloc__(self):
+        del self.stats
+
+    property score:
+        def __get__(self):
+            return self.metric.ComputeScore(self.stats[0])
+
+    property detail:
+        def __get__(self):
+            return self.metric.DetailedScore(self.stats[0]).c_str()
+
+    def __len__(self):
+        return self.stats.size()
+
+    def __iter__(self):
+        for i in range(len(self)):
+            yield self.stats[0][i]
+
+    def __iadd__(SufficientStats self, SufficientStats other):
+        self.stats[0] += other.stats[0]
+        return self
+
+    def __add__(SufficientStats x, SufficientStats y):
+        cdef SufficientStats result = SufficientStats()
+        result.stats = new mteval.SufficientStats(mteval.add(x.stats[0], y.stats[0]))
+        result.metric = x.metric
+        return result
+
+cdef class SegmentEvaluator:
+    cdef shared_ptr[mteval.SegmentEvaluator]* scorer
+    cdef mteval.EvaluationMetric* metric
+    
+    def __dealloc__(self):
+        del self.scorer
+
+    def evaluate(self, sentence):
+        cdef vector[WordID] hyp
+        cdef SufficientStats sf = SufficientStats()
+        sf.metric = self.metric
+        sf.stats = new mteval.SufficientStats()
+        ConvertSentence(string(as_str(sentence.strip())), &hyp)
+        self.scorer.get().Evaluate(hyp, sf.stats)
+        return sf
+
+    def candidate_set(self, Hypergraph hypergraph, unsigned k):
+        cdef mteval.CandidateSet* cs = new mteval.CandidateSet()
+        cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get())
+        cdef Candidate candidate
+        cdef unsigned i
+        for i in range(cs.size()):
+            candidate = Candidate()
+            candidate.candidate = &cs[0][i]
+            candidate.score = self.metric.ComputeScore(cs[0][i].eval_feats)
+            yield candidate
+        del cs
+
+cdef class Scorer:
+    cdef string* name
+
+    def __cinit__(self, char* name):
+        self.name = new string(name)
+    
+    def __call__(self, refs):
+        cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])
+        if isinstance(refs, unicode) or isinstance(refs, str):
+            refs = [refs]
+        cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]()
+        cdef vector[WordID]* refv
+        cdef bytes ref_str
+        for ref in refs:
+            refv = new vector[WordID]()
+            ConvertSentence(string(as_str(ref.strip())), refv)
+            refsv.push_back(refv[0])
+            del refv
+        cdef unsigned i
+        cdef SegmentEvaluator evaluator = SegmentEvaluator()
+        evaluator.metric = metric
+        evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+        del refsv # in theory should not delete but store in SegmentEvaluator
+        return evaluator
+
+    def __str__(self):
+        return self.name.c_str()
+
+BLEU = Scorer('IBM_BLEU')
+TER = Scorer('TER')
diff --git a/python/src/utils.pxd b/python/src/utils.pxd
index 3d518524..94fab5ef 100644
--- a/python/src/utils.pxd
+++ b/python/src/utils.pxd
@@ -50,16 +50,19 @@ cdef extern from "utils/sparse_vector.h":
 
     FastSparseVector[weight_t] operator+(FastSparseVector[weight_t]&, FastSparseVector[weight_t]&)
     FastSparseVector[weight_t] operator-(FastSparseVector[weight_t]&, FastSparseVector[weight_t]&)
+    FastSparseVector[weight_t] operator*(FastSparseVector[weight_t]&, double&)
+    FastSparseVector[weight_t] operator/(FastSparseVector[weight_t]&, double&)
     ostream operator<<(ostream& out, FastSparseVector[weight_t]& v)
 
 cdef extern from "utils/weights.h" namespace "Weights":
     void InitSparseVector(vector[weight_t]& dv, FastSparseVector[weight_t]* sv)
 
-cdef extern from "utils/tdict.cc" namespace "TD":
+cdef extern from "utils/tdict.h" namespace "TD":
     string GetString(vector[WordID]& st)
     unsigned NumWords()
     WordID TDConvert "TD::Convert" (char*)
     char* TDConvert "TD::Convert" (WordID)
+    void ConvertSentence(string& sent, vector[WordID]* ids)
 
 cdef extern from "utils/verbose.h":
     void SetSilent(bint)
@@ -76,3 +79,8 @@ cdef extern from "utils/filelib.h":
 cdef extern from "utils/sampler.h":
     cdef cppclass MT19937:
         pass
+
+cdef extern from "<boost/shared_ptr.hpp>" namespace "boost":
+    cdef cppclass shared_ptr[T]:
+        shared_ptr(shared_ptr& r)
+        T* get()
diff --git a/python/src/vectors.pxi b/python/src/vectors.pxi
index 74d3a0bd..233c9530 100644
--- a/python/src/vectors.pxi
+++ b/python/src/vectors.pxi
@@ -53,7 +53,7 @@ cdef class SparseVector:
             return self.vector.dot((<DenseVector> other).vector[0])
         elif isinstance(other, SparseVector):
             return self.vector.dot((<SparseVector> other).vector[0])
-        raise ValueError('cannot take the dot product of %s and SparseVector' % type(other))
+        raise TypeError('cannot take the dot product of %s and SparseVector' % type(other))
 
     def todense(self):
         cdef DenseVector dense = DenseVector()
@@ -61,11 +61,11 @@ cdef class SparseVector:
         self.vector.init_vector(dense.vector)
         return dense
     
-    def __richcmp__(SparseVector self, SparseVector other, int op):
+    def __richcmp__(SparseVector x, SparseVector y, int op):
         if op == 2: # ==
-            return self.vector[0] == other.vector[0]
+            return x.vector[0] == y.vector[0]
         elif op == 3: # !=
-            return not (self == other)
+            return not (x == y)
         raise NotImplemented('comparison not implemented for SparseVector')
 
     def __len__(self):
@@ -90,12 +90,30 @@ cdef class SparseVector:
         self.vector[0] /= scalar
         return self
 
-    def __add__(SparseVector self, SparseVector other):
+    def __add__(SparseVector x, SparseVector y):
         cdef SparseVector result = SparseVector()
-        result.vector = new FastSparseVector[weight_t](self.vector[0] + other.vector[0])
+        result.vector = new FastSparseVector[weight_t](x.vector[0] + y.vector[0])
         return result
 
-    def __sub__(SparseVector self, SparseVector other):
+    def __sub__(SparseVector x, SparseVector y):
         cdef SparseVector result = SparseVector()
-        result.vector = new FastSparseVector[weight_t](self.vector[0] - other.vector[0])
+        result.vector = new FastSparseVector[weight_t](x.vector[0] - y.vector[0])
+        return result
+
+    def __mul__(x, y):
+        cdef SparseVector vector
+        cdef float scalar
+        if isinstance(x, SparseVector): vector, scalar = x, y
+        else: vector, scalar = y, x
+        cdef SparseVector result = SparseVector()
+        result.vector = new FastSparseVector[weight_t](vector.vector[0] * scalar)
+        return result
+
+    def __div__(x, y):
+        cdef SparseVector vector
+        cdef float scalar
+        if isinstance(x, SparseVector): vector, scalar = x, y
+        else: vector, scalar = y, x
+        cdef SparseVector result = SparseVector()
+        result.vector = new FastSparseVector[weight_t](vector.vector[0] / scalar)
         return result
diff --git a/python/test.py b/python/test.py
index 92403298..b3900eef 100644
--- a/python/test.py
+++ b/python/test.py
@@ -31,9 +31,11 @@ print ' FTree[0]:', forest.viterbi_source_tree().encode('utf8')
 print 'LgProb[0]:', forest.viterbi_features().dot(decoder.weights)
 
 # Get k-best translations
-for i, (sentence, tree) in enumerate(zip(forest.kbest(5), forest.kbest_tree(5)), 1):
+kbest = zip(forest.kbest(5), forest.kbest_tree(5), forest.kbest_features(5))
+for i, (sentence, tree, features) in enumerate(kbest, 1):
     print 'Output[%d]:' % i, sentence.encode('utf8')
     print '  Tree[%d]:' % i, tree.encode('utf8')
+    print ' FVect[%d]:' % i, dict(features)
 
 # Sample translations from the forest
 for sentence in forest.sample(5):