[python] Evaluation metrics in Python

9 files changed, 1405 insertions, 294 deletions

diff --git a/python/cdec/__init__.py b/python/cdec/__init__.py
index 89c323ba..19058493 100644
--- a/python/cdec/__init__.py
+++ b/python/cdec/__init__.py
@@ -1 +1 @@
-from _cdec import Decoder, Lattice, TRule, NT, NTRef
+from _cdec import Decoder, Lattice, TRule, NT, NTRef, ParseFailed, InvalidConfig
diff --git a/python/cdec/score.py b/python/cdec/score.py
index c107446f..d9486ef2 100644
--- a/python/cdec/score.py
+++ b/python/cdec/score.py
@@ -1 +1 @@
-from _cdec import BLEU, TER
+from _cdec import BLEU, TER, Metric
diff --git a/python/src/_cdec.cpp b/python/src/_cdec.cpp
index 3c95ca50..6d704b1d 100644
--- a/python/src/_cdec.cpp
+++ b/python/src/_cdec.cpp
@@ -1,4 +1,4 @@
-/* Generated by Cython 0.16 on Fri Jul 20 18:16:48 2012 */
+/* Generated by Cython 0.16 on Mon Jul 23 19:56:55 2012 */
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
@@ -301,6 +301,7 @@
 #include "observer.h"
 #include "decoder/kbest.h"
 #include "mteval/ns.h"
+#include "py_scorer.h"
 #include "training/candidate_set.h"
 #ifdef _OPENMP
 #include <omp.h>
@@ -423,6 +424,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__;
 struct __pyx_obj_5_cdec___pyx_scope_struct_21___iter__;
 struct __pyx_obj_5_cdec_DenseVector;
 struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__;
+struct __pyx_obj_5_cdec_Metric;
 struct __pyx_obj_5_cdec_SufficientStats;
 struct __pyx_obj_5_cdec___pyx_scope_struct_8_kbest;
 struct __pyx_obj_5_cdec___pyx_scope_struct_10_kbest_features;
@@ -450,16 +452,17 @@ struct __pyx_opt_args_5_cdec_as_str {
   PyObject *error_msg;
 };
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":112
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":117
  *         return CandidateSet(self)
  * 
  * cdef class Scorer:             # <<<<<<<<<<<<<<
  *     cdef string* name
- * 
+ *     cdef mteval.EvaluationMetric* metric
  */
 struct __pyx_obj_5_cdec_Scorer {
   PyObject_HEAD
   std::string *name;
+  EvaluationMetric *metric;
 };
 
 
@@ -561,7 +564,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_2__phrase {
 };
 
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":60
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":65
  *         return result
  * 
  * cdef class CandidateSet:             # <<<<<<<<<<<<<<
@@ -634,7 +637,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_3_genexpr {
 };
 
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":93
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":98
  *         self.cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get())
  * 
  * cdef class SegmentEvaluator:             # <<<<<<<<<<<<<<
@@ -653,7 +656,7 @@ struct __pyx_obj_5_cdec_SegmentEvaluator {
  * 
  *     def __iter__(self):             # <<<<<<<<<<<<<<
  *         for i in range(len(self)):
- *             yield self.stats[0][i]
+ *             yield self[i]
  */
 struct __pyx_obj_5_cdec___pyx_scope_struct_20___iter__ {
   PyObject_HEAD
@@ -850,7 +853,7 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_12___get__ {
 };
 
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":85
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":90
  *         return candidate
  * 
  *     def __iter__(self):             # <<<<<<<<<<<<<<
@@ -898,6 +901,19 @@ struct __pyx_obj_5_cdec___pyx_scope_struct_7___iter__ {
 };
 
 
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":173
+ *         out.fields[i] = ss[i]
+ * 
+ * cdef class Metric:             # <<<<<<<<<<<<<<
+ *     cdef Scorer scorer
+ *     def __cinit__(self):
+ */
+struct __pyx_obj_5_cdec_Metric {
+  PyObject_HEAD
+  struct __pyx_obj_5_cdec_Scorer *scorer;
+};
+
+
 /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":26
  *             return fmap
  * 
@@ -1330,6 +1346,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyBoolOrNull_FromLong(long b) {
 #define __Pyx_PyIter_Next(obj) __Pyx_PyIter_Next2(obj, NULL);
 static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject *, PyObject *); /*proto*/
 
+static CYTHON_INLINE int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /*proto*/
+
 static double __Pyx__PyObject_AsDouble(PyObject* obj); /* proto */
 #define __Pyx_PyObject_AsDouble(obj) \
 ((likely(PyFloat_CheckExact(obj))) ? \
@@ -1523,6 +1541,7 @@ static PyTypeObject *__pyx_ptype_5_cdec_SufficientStats = 0;
 static PyTypeObject *__pyx_ptype_5_cdec_CandidateSet = 0;
 static PyTypeObject *__pyx_ptype_5_cdec_SegmentEvaluator = 0;
 static PyTypeObject *__pyx_ptype_5_cdec_Scorer = 0;
+static PyTypeObject *__pyx_ptype_5_cdec_Metric = 0;
 static PyTypeObject *__pyx_ptype_5_cdec_Decoder = 0;
 static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct____iter__ = 0;
 static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_1___iter__ = 0;
@@ -1551,6 +1570,8 @@ static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_23___cinit__ = 0;
 static PyTypeObject *__pyx_ptype_5_cdec___pyx_scope_struct_24_genexpr = 0;
 static char *__pyx_f_5_cdec_as_str(PyObject *, struct __pyx_opt_args_5_cdec_as_str *__pyx_optional_args); /*proto*/
 static struct __pyx_obj_5_cdec_SufficientStats *__pyx_f_5_cdec_as_stats(PyObject *, PyObject *); /*proto*/
+static float __pyx_f_5_cdec__compute_score(void *, SufficientStats *); /*proto*/
+static void __pyx_f_5_cdec__compute_sufficient_stats(void *, std::string *, std::vector<std::string> *, SufficientStats *); /*proto*/
 #define __Pyx_MODULE_NAME "_cdec"
 int __pyx_module_is_main__cdec = 0;
 
@@ -1676,8 +1697,9 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_5score___get__(struct __pyx_o
 static PyObject *__pyx_pf_5_cdec_15SufficientStats_6detail___get__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self); /* proto */
 static Py_ssize_t __pyx_pf_5_cdec_15SufficientStats_2__len__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_15SufficientStats_4__iter__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self); /* proto */
-static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__iadd__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self, struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_other); /* proto */
-static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /* proto */
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__getitem__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self, int __pyx_v_index); /* proto */
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__iadd__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self, struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_other); /* proto */
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /* proto */
 static int __pyx_pf_5_cdec_12CandidateSet___cinit__(struct __pyx_obj_5_cdec_CandidateSet *__pyx_v_self, struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_evaluator); /* proto */
 static void __pyx_pf_5_cdec_12CandidateSet_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_CandidateSet *__pyx_v_self); /* proto */
 static Py_ssize_t __pyx_pf_5_cdec_12CandidateSet_4__len__(struct __pyx_obj_5_cdec_CandidateSet *__pyx_v_self); /* proto */
@@ -1687,10 +1709,14 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_11add_kbest(struct __pyx_obj_5_c
 static void __pyx_pf_5_cdec_16SegmentEvaluator___dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self, PyObject *__pyx_v_sentence); /* proto */
 static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(struct __pyx_obj_5_cdec_SegmentEvaluator *__pyx_v_self); /* proto */
-static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, char *__pyx_v_name); /* proto */
+static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, PyObject *__pyx_v_name); /* proto */
 static void __pyx_pf_5_cdec_6Scorer_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_5_cdec_Scorer *__pyx_v_self); /* proto */
 static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, PyObject *__pyx_v_refs); /* proto */
 static PyObject *__pyx_pf_5_cdec_6Scorer_6__str__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self); /* proto */
+static int __pyx_pf_5_cdec_6Metric___cinit__(struct __pyx_obj_5_cdec_Metric *__pyx_v_self); /* proto */
+static PyObject *__pyx_pf_5_cdec_6Metric_2__call__(struct __pyx_obj_5_cdec_Metric *__pyx_v_self, PyObject *__pyx_v_refs); /* proto */
+static PyObject *__pyx_pf_5_cdec_6Metric_4score(CYTHON_UNUSED struct __pyx_obj_5_cdec_Metric *__pyx_v_stats); /* proto */
+static PyObject *__pyx_pf_5_cdec_6Metric_6evaluate(CYTHON_UNUSED struct __pyx_obj_5_cdec_Metric *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v_hyp, CYTHON_UNUSED PyObject *__pyx_v_refs); /* proto */
 static PyObject *__pyx_pf_5_cdec_2_make_config(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_config); /* proto */
 static PyObject *__pyx_pf_5_cdec_7Decoder_9__cinit___genexpr(PyObject *__pyx_self); /* proto */
 static int __pyx_pf_5_cdec_7Decoder___cinit__(struct __pyx_obj_5_cdec_Decoder *__pyx_v_self, PyObject *__pyx_v_config_str, PyObject *__pyx_v_config); /* proto */
@@ -1726,15 +1752,16 @@ static char __pyx_k_27[] = "digraph lattice {";
 static char __pyx_k_35[] = "}";
 static char __pyx_k_38[] = "/Users/vchahun/Sandbox/cdec/python/src/lattice.pxi";
 static char __pyx_k_39[] = "\n";
-static char __pyx_k_41[] = "candidate set index out of range";
-static char __pyx_k_43[] = "%s %s";
-static char __pyx_k_44[] = "%s = %s";
-static char __pyx_k_46[] = "formalism \"%s\" unknown";
-static char __pyx_k_47[] = "cannot initialize weights with %s";
-static char __pyx_k_48[] = "#";
-static char __pyx_k_51[] = "Cannot translate input type %s";
-static char __pyx_k_54[] = "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi";
-static char __pyx_k_59[] = "/Users/vchahun/Sandbox/cdec/python/src/_cdec.pyx";
+static char __pyx_k_41[] = "sufficient stats vector index out of range";
+static char __pyx_k_43[] = "candidate set index out of range";
+static char __pyx_k_45[] = "%s %s";
+static char __pyx_k_46[] = "%s = %s";
+static char __pyx_k_48[] = "formalism \"%s\" unknown";
+static char __pyx_k_49[] = "cannot initialize weights with %s";
+static char __pyx_k_50[] = "#";
+static char __pyx_k_53[] = "Cannot translate input type %s";
+static char __pyx_k_56[] = "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi";
+static char __pyx_k_61[] = "/Users/vchahun/Sandbox/cdec/python/src/_cdec.pyx";
 static char __pyx_k__a[] = "a";
 static char __pyx_k__e[] = "e";
 static char __pyx_k__f[] = "f";
@@ -1746,6 +1773,7 @@ static char __pyx_k__cat[] = "cat";
 static char __pyx_k__dot[] = "dot";
 static char __pyx_k__fst[] = "fst";
 static char __pyx_k__get[] = "get";
+static char __pyx_k__hyp[] = "hyp";
 static char __pyx_k__inp[] = "inp";
 static char __pyx_k__key[] = "key";
 static char __pyx_k__lhs[] = "lhs";
@@ -1769,6 +1797,7 @@ static char __pyx_k__label[] = "label";
 static char __pyx_k__lines[] = "lines";
 static char __pyx_k__range[] = "range";
 static char __pyx_k__rules[] = "rules";
+static char __pyx_k__score[] = "score";
 static char __pyx_k__split[] = "split";
 static char __pyx_k__strip[] = "strip";
 static char __pyx_k__value[] = "value";
@@ -1788,14 +1817,17 @@ static char __pyx_k__IBM_BLEU[] = "IBM_BLEU";
 static char __pyx_k__KeyError[] = "KeyError";
 static char __pyx_k____exit__[] = "__exit__";
 static char __pyx_k____main__[] = "__main__";
+static char __pyx_k____name__[] = "__name__";
 static char __pyx_k____test__[] = "__test__";
 static char __pyx_k__encoding[] = "encoding";
+static char __pyx_k__evaluate[] = "evaluate";
 static char __pyx_k__in_edges[] = "in_edges";
 static char __pyx_k__lexalign[] = "lexalign";
 static char __pyx_k__lextrans[] = "lextrans";
 static char __pyx_k__sentence[] = "sentence";
 static char __pyx_k__Exception[] = "Exception";
 static char __pyx_k__TypeError[] = "TypeError";
+static char __pyx_k____class__[] = "__class__";
 static char __pyx_k____enter__[] = "__enter__";
 static char __pyx_k__enumerate[] = "enumerate";
 static char __pyx_k__evaluator[] = "evaluator";
@@ -1835,13 +1867,14 @@ static PyObject *__pyx_kp_s_39;
 static PyObject *__pyx_kp_s_4;
 static PyObject *__pyx_kp_s_41;
 static PyObject *__pyx_kp_s_43;
-static PyObject *__pyx_kp_s_44;
+static PyObject *__pyx_kp_s_45;
 static PyObject *__pyx_kp_s_46;
-static PyObject *__pyx_kp_s_47;
 static PyObject *__pyx_kp_s_48;
-static PyObject *__pyx_kp_s_51;
-static PyObject *__pyx_kp_s_54;
-static PyObject *__pyx_kp_s_59;
+static PyObject *__pyx_kp_s_49;
+static PyObject *__pyx_kp_s_50;
+static PyObject *__pyx_kp_s_53;
+static PyObject *__pyx_kp_s_56;
+static PyObject *__pyx_kp_s_61;
 static PyObject *__pyx_kp_s_7;
 static PyObject *__pyx_kp_s_8;
 static PyObject *__pyx_kp_s_9;
@@ -1856,9 +1889,11 @@ static PyObject *__pyx_n_s__ParseFailed;
 static PyObject *__pyx_n_s__TER;
 static PyObject *__pyx_n_s__TypeError;
 static PyObject *__pyx_n_s__ValueError;
+static PyObject *__pyx_n_s____class__;
 static PyObject *__pyx_n_s____enter__;
 static PyObject *__pyx_n_s____exit__;
 static PyObject *__pyx_n_s____main__;
+static PyObject *__pyx_n_s____name__;
 static PyObject *__pyx_n_s____test__;
 static PyObject *__pyx_n_s___cdec;
 static PyObject *__pyx_n_s___make_config;
@@ -1877,6 +1912,7 @@ static PyObject *__pyx_n_s__encode;
 static PyObject *__pyx_n_s__encoding;
 static PyObject *__pyx_n_s__enumerate;
 static PyObject *__pyx_n_s__eval;
+static PyObject *__pyx_n_s__evaluate;
 static PyObject *__pyx_n_s__evaluator;
 static PyObject *__pyx_n_s__f;
 static PyObject *__pyx_n_s__formalism;
@@ -1884,6 +1920,7 @@ static PyObject *__pyx_n_s__fst;
 static PyObject *__pyx_n_s__genexpr;
 static PyObject *__pyx_n_s__get;
 static PyObject *__pyx_n_s__grammar;
+static PyObject *__pyx_n_s__hyp;
 static PyObject *__pyx_n_s__hypergraph;
 static PyObject *__pyx_n_s__i;
 static PyObject *__pyx_n_s__in_edges;
@@ -1909,6 +1946,7 @@ static PyObject *__pyx_n_s__refs;
 static PyObject *__pyx_n_s__replace;
 static PyObject *__pyx_n_s__rules;
 static PyObject *__pyx_n_s__scfg;
+static PyObject *__pyx_n_s__score;
 static PyObject *__pyx_n_s__scores;
 static PyObject *__pyx_n_s__self;
 static PyObject *__pyx_n_s__sentence;
@@ -1936,16 +1974,17 @@ static PyObject *__pyx_k_tuple_33;
 static PyObject *__pyx_k_tuple_36;
 static PyObject *__pyx_k_tuple_40;
 static PyObject *__pyx_k_tuple_42;
-static PyObject *__pyx_k_tuple_45;
-static PyObject *__pyx_k_tuple_49;
-static PyObject *__pyx_k_tuple_50;
+static PyObject *__pyx_k_tuple_44;
+static PyObject *__pyx_k_tuple_47;
+static PyObject *__pyx_k_tuple_51;
 static PyObject *__pyx_k_tuple_52;
-static PyObject *__pyx_k_tuple_55;
-static PyObject *__pyx_k_tuple_56;
+static PyObject *__pyx_k_tuple_54;
 static PyObject *__pyx_k_tuple_57;
+static PyObject *__pyx_k_tuple_58;
+static PyObject *__pyx_k_tuple_59;
 static PyObject *__pyx_k_codeobj_37;
-static PyObject *__pyx_k_codeobj_53;
-static PyObject *__pyx_k_codeobj_58;
+static PyObject *__pyx_k_codeobj_55;
+static PyObject *__pyx_k_codeobj_60;
 
 /* "_cdec.pyx":6
  * cimport decoder
@@ -7390,7 +7429,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG
  *         for trule in rules:
  *             if not isinstance(trule, BaseTRule):             # <<<<<<<<<<<<<<
  *                 raise ValueError('the grammar should contain TRule objects')
- *             _g.AddRule((<TRule> trule).rule[0])
+ *             _g.AddRule((<BaseTRule> trule).rule[0])
  */
     __pyx_t_4 = ((PyObject *)((PyObject*)__pyx_ptype_5_cdec_BaseTRule));
     __Pyx_INCREF(__pyx_t_4);
@@ -7403,7 +7442,7 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG
  *         for trule in rules:
  *             if not isinstance(trule, BaseTRule):
  *                 raise ValueError('the grammar should contain TRule objects')             # <<<<<<<<<<<<<<
- *             _g.AddRule((<TRule> trule).rule[0])
+ *             _g.AddRule((<BaseTRule> trule).rule[0])
  */
       __pyx_t_4 = PyObject_Call(__pyx_builtin_ValueError, ((PyObject *)__pyx_k_tuple_14), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_4);
@@ -7417,9 +7456,9 @@ static int __pyx_pf_5_cdec_11TextGrammar___cinit__(struct __pyx_obj_5_cdec_TextG
     /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":176
  *             if not isinstance(trule, BaseTRule):
  *                 raise ValueError('the grammar should contain TRule objects')
- *             _g.AddRule((<TRule> trule).rule[0])             # <<<<<<<<<<<<<<
+ *             _g.AddRule((<BaseTRule> trule).rule[0])             # <<<<<<<<<<<<<<
  */
-    __pyx_v__g->AddRule((((struct __pyx_obj_5_cdec_TRule *)__pyx_v_trule)->__pyx_base.rule[0]));
+    __pyx_v__g->AddRule((((struct __pyx_obj_5_cdec_BaseTRule *)__pyx_v_trule)->rule[0]));
   }
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
 
@@ -13071,7 +13110,7 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_5__iter__(PyObject *__pyx_v_s
  * 
  *     def __iter__(self):             # <<<<<<<<<<<<<<
  *         for i in range(len(self)):
- *             yield self.stats[0][i]
+ *             yield self[i]
  */
 
 static PyObject *__pyx_pf_5_cdec_15SufficientStats_4__iter__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self) {
@@ -13118,7 +13157,6 @@ static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorO
   PyObject *__pyx_t_2 = NULL;
   PyObject *__pyx_t_3 = NULL;
   PyObject *(*__pyx_t_4)(PyObject *);
-  unsigned int __pyx_t_5;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("None", 0);
   switch (__pyx_generator->resume_label) {
@@ -13135,7 +13173,7 @@ static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorO
  * 
  *     def __iter__(self):
  *         for i in range(len(self)):             # <<<<<<<<<<<<<<
- *             yield self.stats[0][i]
+ *             yield self[i]
  * 
  */
   __pyx_t_1 = PyObject_Length(((PyObject *)__pyx_cur_scope->__pyx_v_self)); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 45; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
@@ -13185,12 +13223,11 @@ static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorO
     /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":46
  *     def __iter__(self):
  *         for i in range(len(self)):
- *             yield self.stats[0][i]             # <<<<<<<<<<<<<<
+ *             yield self[i]             # <<<<<<<<<<<<<<
  * 
- *     def __iadd__(SufficientStats self, SufficientStats other):
+ *     def __getitem__(self, int index):
  */
-    __pyx_t_5 = __Pyx_PyInt_AsUnsignedInt(__pyx_cur_scope->__pyx_v_i); if (unlikely((__pyx_t_5 == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __pyx_t_2 = PyFloat_FromDouble(((__pyx_cur_scope->__pyx_v_self->stats[0])[__pyx_t_5])); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_2 = PyObject_GetItem(((PyObject *)__pyx_cur_scope->__pyx_v_self), __pyx_cur_scope->__pyx_v_i); if (!__pyx_t_2) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_2);
     __pyx_r = __pyx_t_2;
     __pyx_t_2 = 0;
@@ -13226,13 +13263,111 @@ static PyObject *__pyx_gb_5_cdec_15SufficientStats_6generator14(__pyx_GeneratorO
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_15SufficientStats_8__iadd__(PyObject *__pyx_v_self, PyObject *__pyx_v_other); /*proto*/
-static PyObject *__pyx_pw_5_cdec_15SufficientStats_8__iadd__(PyObject *__pyx_v_self, PyObject *__pyx_v_other) {
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_8__getitem__(PyObject *__pyx_v_self, PyObject *__pyx_arg_index); /*proto*/
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_8__getitem__(PyObject *__pyx_v_self, PyObject *__pyx_arg_index) {
+  int __pyx_v_index;
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0);
+  assert(__pyx_arg_index); {
+    __pyx_v_index = __Pyx_PyInt_AsInt(__pyx_arg_index); if (unlikely((__pyx_v_index == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 48; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("_cdec.SufficientStats.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_7__getitem__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self), ((int)__pyx_v_index));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":48
+ *             yield self[i]
+ * 
+ *     def __getitem__(self, int index):             # <<<<<<<<<<<<<<
+ *         if not 0 <= index < len(self):
+ *             raise IndexError('sufficient stats vector index out of range')
+ */
+
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__getitem__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self, int __pyx_v_index) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  int __pyx_t_1;
+  Py_ssize_t __pyx_t_2;
+  int __pyx_t_3;
+  PyObject *__pyx_t_4 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__getitem__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":49
+ * 
+ *     def __getitem__(self, int index):
+ *         if not 0 <= index < len(self):             # <<<<<<<<<<<<<<
+ *             raise IndexError('sufficient stats vector index out of range')
+ *         return self.stats[0][index]
+ */
+  __pyx_t_1 = (0 <= __pyx_v_index);
+  if (__pyx_t_1) {
+    __pyx_t_2 = PyObject_Length(((PyObject *)__pyx_v_self)); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 49; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = (__pyx_v_index < __pyx_t_2);
+  }
+  __pyx_t_3 = (!__pyx_t_1);
+  if (__pyx_t_3) {
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":50
+ *     def __getitem__(self, int index):
+ *         if not 0 <= index < len(self):
+ *             raise IndexError('sufficient stats vector index out of range')             # <<<<<<<<<<<<<<
+ *         return self.stats[0][index]
+ * 
+ */
+    __pyx_t_4 = PyObject_Call(__pyx_builtin_IndexError, ((PyObject *)__pyx_k_tuple_42), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_4);
+    __Pyx_Raise(__pyx_t_4, 0, 0, 0);
+    __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
+    {__pyx_filename = __pyx_f[5]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    goto __pyx_L3;
+  }
+  __pyx_L3:;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":51
+ *         if not 0 <= index < len(self):
+ *             raise IndexError('sufficient stats vector index out of range')
+ *         return self.stats[0][index]             # <<<<<<<<<<<<<<
+ * 
+ *     def __iadd__(SufficientStats self, SufficientStats other):
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_4 = PyFloat_FromDouble(((__pyx_v_self->stats[0])[__pyx_v_index])); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_4);
+  __pyx_r = __pyx_t_4;
+  __pyx_t_4 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_4);
+  __Pyx_AddTraceback("_cdec.SufficientStats.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_10__iadd__(PyObject *__pyx_v_self, PyObject *__pyx_v_other); /*proto*/
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_10__iadd__(PyObject *__pyx_v_self, PyObject *__pyx_v_other) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__iadd__ (wrapper)", 0);
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5_cdec_SufficientStats, 1, "other", 0))) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 48; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_7__iadd__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self), ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_other));
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_other), __pyx_ptype_5_cdec_SufficientStats, 1, "other", 0))) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_9__iadd__(((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_self), ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_v_other));
   goto __pyx_L0;
   __pyx_L1_error:;
   __pyx_r = NULL;
@@ -13241,20 +13376,20 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_8__iadd__(PyObject *__pyx_v_s
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":48
- *             yield self.stats[0][i]
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":53
+ *         return self.stats[0][index]
  * 
  *     def __iadd__(SufficientStats self, SufficientStats other):             # <<<<<<<<<<<<<<
  *         self.stats[0] += other.stats[0]
  *         return self
  */
 
-static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__iadd__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self, struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_other) {
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__iadd__(struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_self, struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_other) {
   PyObject *__pyx_r = NULL;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__iadd__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":49
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":54
  * 
  *     def __iadd__(SufficientStats self, SufficientStats other):
  *         self.stats[0] += other.stats[0]             # <<<<<<<<<<<<<<
@@ -13263,7 +13398,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__iadd__(struct __pyx_obj_5_
  */
   (__pyx_v_self->stats[0]) += (__pyx_v_other->stats[0]);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":50
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":55
  *     def __iadd__(SufficientStats self, SufficientStats other):
  *         self.stats[0] += other.stats[0]
  *         return self             # <<<<<<<<<<<<<<
@@ -13283,17 +13418,17 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_7__iadd__(struct __pyx_obj_5_
 }
 
 /* Python wrapper */
-static PyObject *__pyx_pw_5_cdec_15SufficientStats_10__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /*proto*/
-static PyObject *__pyx_pw_5_cdec_15SufficientStats_10__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_12__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y); /*proto*/
+static PyObject *__pyx_pw_5_cdec_15SufficientStats_12__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
   PyObject *__pyx_r = 0;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__add__ (wrapper)", 0);
-  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_9__add__(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y));
+  __pyx_r = __pyx_pf_5_cdec_15SufficientStats_11__add__(((PyObject *)__pyx_v_x), ((PyObject *)__pyx_v_y));
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":52
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":57
  *         return self
  * 
  *     def __add__(x, y):             # <<<<<<<<<<<<<<
@@ -13301,7 +13436,7 @@ static PyObject *__pyx_pw_5_cdec_15SufficientStats_10__add__(PyObject *__pyx_v_x
  *         cdef SufficientStats sy = as_stats(y, x)
  */
 
-static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
+static PyObject *__pyx_pf_5_cdec_15SufficientStats_11__add__(PyObject *__pyx_v_x, PyObject *__pyx_v_y) {
   struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_sx = 0;
   struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_sy = 0;
   struct __pyx_obj_5_cdec_SufficientStats *__pyx_v_result = 0;
@@ -13313,43 +13448,43 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__add__(PyObject *__pyx_v_x,
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__add__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":53
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":58
  * 
  *     def __add__(x, y):
  *         cdef SufficientStats sx = as_stats(x, y)             # <<<<<<<<<<<<<<
  *         cdef SufficientStats sy = as_stats(y, x)
  *         cdef SufficientStats result = SufficientStats()
  */
-  __pyx_t_1 = ((PyObject *)__pyx_f_5_cdec_as_stats(__pyx_v_x, __pyx_v_y)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = ((PyObject *)__pyx_f_5_cdec_as_stats(__pyx_v_x, __pyx_v_y)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __pyx_v_sx = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1);
   __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":54
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":59
  *     def __add__(x, y):
  *         cdef SufficientStats sx = as_stats(x, y)
  *         cdef SufficientStats sy = as_stats(y, x)             # <<<<<<<<<<<<<<
  *         cdef SufficientStats result = SufficientStats()
  *         result.stats = new mteval.SufficientStats(mteval.add(sx.stats[0], sy.stats[0]))
  */
-  __pyx_t_1 = ((PyObject *)__pyx_f_5_cdec_as_stats(__pyx_v_y, __pyx_v_x)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = ((PyObject *)__pyx_f_5_cdec_as_stats(__pyx_v_y, __pyx_v_x)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __pyx_v_sy = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1);
   __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":55
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":60
  *         cdef SufficientStats sx = as_stats(x, y)
  *         cdef SufficientStats sy = as_stats(y, x)
  *         cdef SufficientStats result = SufficientStats()             # <<<<<<<<<<<<<<
  *         result.stats = new mteval.SufficientStats(mteval.add(sx.stats[0], sy.stats[0]))
  *         result.metric = sx.metric
  */
-  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SufficientStats)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 55; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SufficientStats)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __pyx_v_result = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1);
   __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":56
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":61
  *         cdef SufficientStats sy = as_stats(y, x)
  *         cdef SufficientStats result = SufficientStats()
  *         result.stats = new mteval.SufficientStats(mteval.add(sx.stats[0], sy.stats[0]))             # <<<<<<<<<<<<<<
@@ -13358,7 +13493,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__add__(PyObject *__pyx_v_x,
  */
   __pyx_v_result->stats = new SufficientStats(operator+((__pyx_v_sx->stats[0]), (__pyx_v_sy->stats[0])));
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":57
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":62
  *         cdef SufficientStats result = SufficientStats()
  *         result.stats = new mteval.SufficientStats(mteval.add(sx.stats[0], sy.stats[0]))
  *         result.metric = sx.metric             # <<<<<<<<<<<<<<
@@ -13367,7 +13502,7 @@ static PyObject *__pyx_pf_5_cdec_15SufficientStats_9__add__(PyObject *__pyx_v_x,
  */
   __pyx_v_result->metric = __pyx_v_sx->metric;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":58
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":63
  *         result.stats = new mteval.SufficientStats(mteval.add(sx.stats[0], sy.stats[0]))
  *         result.metric = sx.metric
  *         return result             # <<<<<<<<<<<<<<
@@ -13420,7 +13555,7 @@ static int __pyx_pw_5_cdec_12CandidateSet_1__cinit__(PyObject *__pyx_v_self, PyO
         else goto __pyx_L5_argtuple_error;
       }
       if (unlikely(kw_args > 0)) {
-        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
       }
     } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
       goto __pyx_L5_argtuple_error;
@@ -13431,13 +13566,13 @@ static int __pyx_pw_5_cdec_12CandidateSet_1__cinit__(PyObject *__pyx_v_self, PyO
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L5_argtuple_error:;
-  __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   __pyx_L3_error:;
   __Pyx_AddTraceback("_cdec.CandidateSet.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __Pyx_RefNannyFinishContext();
   return -1;
   __pyx_L4_argument_unpacking_done:;
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_evaluator), __pyx_ptype_5_cdec_SegmentEvaluator, 1, "evaluator", 0))) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_evaluator), __pyx_ptype_5_cdec_SegmentEvaluator, 1, "evaluator", 0))) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_r = __pyx_pf_5_cdec_12CandidateSet___cinit__(((struct __pyx_obj_5_cdec_CandidateSet *)__pyx_v_self), __pyx_v_evaluator);
   goto __pyx_L0;
   __pyx_L1_error:;
@@ -13447,7 +13582,7 @@ static int __pyx_pw_5_cdec_12CandidateSet_1__cinit__(PyObject *__pyx_v_self, PyO
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":65
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":70
  *     cdef mteval.CandidateSet* cs
  * 
  *     def __cinit__(self, SegmentEvaluator evaluator):             # <<<<<<<<<<<<<<
@@ -13460,7 +13595,7 @@ static int __pyx_pf_5_cdec_12CandidateSet___cinit__(struct __pyx_obj_5_cdec_Cand
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__cinit__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":66
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":71
  * 
  *     def __cinit__(self, SegmentEvaluator evaluator):
  *         self.scorer = new shared_ptr[mteval.SegmentEvaluator](evaluator.scorer[0])             # <<<<<<<<<<<<<<
@@ -13469,7 +13604,7 @@ static int __pyx_pf_5_cdec_12CandidateSet___cinit__(struct __pyx_obj_5_cdec_Cand
  */
   __pyx_v_self->scorer = new boost::shared_ptr<SegmentEvaluator>((__pyx_v_evaluator->scorer[0]));
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":67
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":72
  *     def __cinit__(self, SegmentEvaluator evaluator):
  *         self.scorer = new shared_ptr[mteval.SegmentEvaluator](evaluator.scorer[0])
  *         self.metric = evaluator.metric             # <<<<<<<<<<<<<<
@@ -13478,7 +13613,7 @@ static int __pyx_pf_5_cdec_12CandidateSet___cinit__(struct __pyx_obj_5_cdec_Cand
  */
   __pyx_v_self->metric = __pyx_v_evaluator->metric;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":68
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":73
  *         self.scorer = new shared_ptr[mteval.SegmentEvaluator](evaluator.scorer[0])
  *         self.metric = evaluator.metric
  *         self.cs = new mteval.CandidateSet()             # <<<<<<<<<<<<<<
@@ -13501,7 +13636,7 @@ static void __pyx_pw_5_cdec_12CandidateSet_3__dealloc__(PyObject *__pyx_v_self)
   __Pyx_RefNannyFinishContext();
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":70
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":75
  *         self.cs = new mteval.CandidateSet()
  * 
  *     def __dealloc__(self):             # <<<<<<<<<<<<<<
@@ -13513,7 +13648,7 @@ static void __pyx_pf_5_cdec_12CandidateSet_2__dealloc__(CYTHON_UNUSED struct __p
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__dealloc__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":71
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":76
  * 
  *     def __dealloc__(self):
  *         del self.scorer             # <<<<<<<<<<<<<<
@@ -13522,7 +13657,7 @@ static void __pyx_pf_5_cdec_12CandidateSet_2__dealloc__(CYTHON_UNUSED struct __p
  */
   delete __pyx_v_self->scorer;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":72
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":77
  *     def __dealloc__(self):
  *         del self.scorer
  *         del self.cs             # <<<<<<<<<<<<<<
@@ -13545,7 +13680,7 @@ static Py_ssize_t __pyx_pw_5_cdec_12CandidateSet_5__len__(PyObject *__pyx_v_self
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":74
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":79
  *         del self.cs
  * 
  *     def __len__(self):             # <<<<<<<<<<<<<<
@@ -13558,7 +13693,7 @@ static Py_ssize_t __pyx_pf_5_cdec_12CandidateSet_4__len__(struct __pyx_obj_5_cde
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__len__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":75
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":80
  * 
  *     def __len__(self):
  *         return self.cs.size()             # <<<<<<<<<<<<<<
@@ -13582,7 +13717,7 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_7__getitem__(PyObject *__pyx_v_s
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0);
   assert(__pyx_arg_k); {
-    __pyx_v_k = __Pyx_PyInt_AsInt(__pyx_arg_k); if (unlikely((__pyx_v_k == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 77; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    __pyx_v_k = __Pyx_PyInt_AsInt(__pyx_arg_k); if (unlikely((__pyx_v_k == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 82; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L3_error:;
@@ -13595,7 +13730,7 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_7__getitem__(PyObject *__pyx_v_s
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":77
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":82
  *         return self.cs.size()
  * 
  *     def __getitem__(self,int k):             # <<<<<<<<<<<<<<
@@ -13615,7 +13750,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__getitem__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":78
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":83
  * 
  *     def __getitem__(self,int k):
  *         if not 0 <= k < self.cs.size():             # <<<<<<<<<<<<<<
@@ -13629,35 +13764,35 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_
   __pyx_t_2 = (!__pyx_t_1);
   if (__pyx_t_2) {
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":79
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":84
  *     def __getitem__(self,int k):
  *         if not 0 <= k < self.cs.size():
  *             raise IndexError('candidate set index out of range')             # <<<<<<<<<<<<<<
  *         cdef Candidate candidate = Candidate()
  *         candidate.candidate = &self.cs[0][k]
  */
-    __pyx_t_3 = PyObject_Call(__pyx_builtin_IndexError, ((PyObject *)__pyx_k_tuple_42), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 79; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = PyObject_Call(__pyx_builtin_IndexError, ((PyObject *)__pyx_k_tuple_44), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
     __Pyx_Raise(__pyx_t_3, 0, 0, 0);
     __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
-    {__pyx_filename = __pyx_f[5]; __pyx_lineno = 79; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    {__pyx_filename = __pyx_f[5]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     goto __pyx_L3;
   }
   __pyx_L3:;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":80
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":85
  *         if not 0 <= k < self.cs.size():
  *             raise IndexError('candidate set index out of range')
  *         cdef Candidate candidate = Candidate()             # <<<<<<<<<<<<<<
  *         candidate.candidate = &self.cs[0][k]
  *         candidate.score = self.metric.ComputeScore(self.cs[0][k].eval_feats)
  */
-  __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Candidate)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 80; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Candidate)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 85; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_3);
   __pyx_v_candidate = ((struct __pyx_obj_5_cdec_Candidate *)__pyx_t_3);
   __pyx_t_3 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":81
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":86
  *             raise IndexError('candidate set index out of range')
  *         cdef Candidate candidate = Candidate()
  *         candidate.candidate = &self.cs[0][k]             # <<<<<<<<<<<<<<
@@ -13666,7 +13801,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_
  */
   __pyx_v_candidate->candidate = (&((__pyx_v_self->cs[0])[__pyx_v_k]));
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":82
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":87
  *         cdef Candidate candidate = Candidate()
  *         candidate.candidate = &self.cs[0][k]
  *         candidate.score = self.metric.ComputeScore(self.cs[0][k].eval_feats)             # <<<<<<<<<<<<<<
@@ -13675,7 +13810,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_6__getitem__(struct __pyx_obj_5_
  */
   __pyx_v_candidate->score = __pyx_v_self->metric->ComputeScore(((__pyx_v_self->cs[0])[__pyx_v_k]).eval_feats);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":83
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":88
  *         candidate.candidate = &self.cs[0][k]
  *         candidate.score = self.metric.ComputeScore(self.cs[0][k].eval_feats)
  *         return candidate             # <<<<<<<<<<<<<<
@@ -13712,7 +13847,7 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_9__iter__(PyObject *__pyx_v_self
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":85
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":90
  *         return candidate
  * 
  *     def __iter__(self):             # <<<<<<<<<<<<<<
@@ -13738,7 +13873,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_8__iter__(struct __pyx_obj_5_cde
   __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
   __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
   {
-    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_12CandidateSet_10generator15, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 85; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_5_cdec_12CandidateSet_10generator15, (PyObject *) __pyx_cur_scope); if (unlikely(!gen)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_DECREF(__pyx_cur_scope);
     __Pyx_RefNannyFinishContext();
     return (PyObject *) gen;
@@ -13773,27 +13908,27 @@ static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator15(__pyx_GeneratorObj
     return NULL;
   }
   __pyx_L3_first_run:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 85; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":87
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":92
  *     def __iter__(self):
  *         cdef unsigned i
  *         for i in range(len(self)):             # <<<<<<<<<<<<<<
  *             yield self[i]
  * 
  */
-  __pyx_t_1 = PyObject_Length(((PyObject *)__pyx_cur_scope->__pyx_v_self)); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 87; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_Length(((PyObject *)__pyx_cur_scope->__pyx_v_self)); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 92; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) {
     __pyx_cur_scope->__pyx_v_i = __pyx_t_2;
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":88
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":93
  *         cdef unsigned i
  *         for i in range(len(self)):
  *             yield self[i]             # <<<<<<<<<<<<<<
  * 
  *     def add_kbest(self, Hypergraph hypergraph, unsigned k):
  */
-    __pyx_t_3 = __Pyx_GetItemInt(((PyObject *)__pyx_cur_scope->__pyx_v_self), __pyx_cur_scope->__pyx_v_i, sizeof(unsigned int)+1, PyLong_FromUnsignedLong); if (!__pyx_t_3) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = __Pyx_GetItemInt(((PyObject *)__pyx_cur_scope->__pyx_v_self), __pyx_cur_scope->__pyx_v_i, sizeof(unsigned int)+1, PyLong_FromUnsignedLong); if (!__pyx_t_3) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
     __pyx_r = __pyx_t_3;
     __pyx_t_3 = 0;
@@ -13807,7 +13942,7 @@ static PyObject *__pyx_gb_5_cdec_12CandidateSet_10generator15(__pyx_GeneratorObj
     __pyx_L6_resume_from_yield:;
     __pyx_t_1 = __pyx_cur_scope->__pyx_t_0;
     __pyx_t_2 = __pyx_cur_scope->__pyx_t_1;
-    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
   PyErr_SetNone(PyExc_StopIteration);
   goto __pyx_L0;
@@ -13851,11 +13986,11 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_12add_kbest(PyObject *__pyx_v_se
         values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__k);
         if (likely(values[1])) kw_args--;
         else {
-          __Pyx_RaiseArgtupleInvalid("add_kbest", 1, 2, 2, 1); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+          __Pyx_RaiseArgtupleInvalid("add_kbest", 1, 2, 2, 1); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
         }
       }
       if (unlikely(kw_args > 0)) {
-        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "add_kbest") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "add_kbest") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
       }
     } else if (PyTuple_GET_SIZE(__pyx_args) != 2) {
       goto __pyx_L5_argtuple_error;
@@ -13864,17 +13999,17 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_12add_kbest(PyObject *__pyx_v_se
       values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
     }
     __pyx_v_hypergraph = ((struct __pyx_obj_5_cdec_Hypergraph *)values[0]);
-    __pyx_v_k = __Pyx_PyInt_AsUnsignedInt(values[1]); if (unlikely((__pyx_v_k == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    __pyx_v_k = __Pyx_PyInt_AsUnsignedInt(values[1]); if (unlikely((__pyx_v_k == (unsigned int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L5_argtuple_error:;
-  __Pyx_RaiseArgtupleInvalid("add_kbest", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __Pyx_RaiseArgtupleInvalid("add_kbest", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   __pyx_L3_error:;
   __Pyx_AddTraceback("_cdec.CandidateSet.add_kbest", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __Pyx_RefNannyFinishContext();
   return NULL;
   __pyx_L4_argument_unpacking_done:;
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_hypergraph), __pyx_ptype_5_cdec_Hypergraph, 1, "hypergraph", 0))) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_hypergraph), __pyx_ptype_5_cdec_Hypergraph, 1, "hypergraph", 0))) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 95; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_r = __pyx_pf_5_cdec_12CandidateSet_11add_kbest(((struct __pyx_obj_5_cdec_CandidateSet *)__pyx_v_self), __pyx_v_hypergraph, __pyx_v_k);
   goto __pyx_L0;
   __pyx_L1_error:;
@@ -13884,7 +14019,7 @@ static PyObject *__pyx_pw_5_cdec_12CandidateSet_12add_kbest(PyObject *__pyx_v_se
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":90
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":95
  *             yield self[i]
  * 
  *     def add_kbest(self, Hypergraph hypergraph, unsigned k):             # <<<<<<<<<<<<<<
@@ -13897,7 +14032,7 @@ static PyObject *__pyx_pf_5_cdec_12CandidateSet_11add_kbest(struct __pyx_obj_5_c
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("add_kbest", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":91
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":96
  * 
  *     def add_kbest(self, Hypergraph hypergraph, unsigned k):
  *         self.cs.AddKBestCandidates(hypergraph.hg[0], k, self.scorer.get())             # <<<<<<<<<<<<<<
@@ -13921,7 +14056,7 @@ static void __pyx_pw_5_cdec_16SegmentEvaluator_1__dealloc__(PyObject *__pyx_v_se
   __Pyx_RefNannyFinishContext();
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":97
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":102
  *     cdef mteval.EvaluationMetric* metric
  * 
  *     def __dealloc__(self):             # <<<<<<<<<<<<<<
@@ -13933,7 +14068,7 @@ static void __pyx_pf_5_cdec_16SegmentEvaluator___dealloc__(CYTHON_UNUSED struct
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__dealloc__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":98
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":103
  * 
  *     def __dealloc__(self):
  *         del self.scorer             # <<<<<<<<<<<<<<
@@ -13956,7 +14091,7 @@ static PyObject *__pyx_pw_5_cdec_16SegmentEvaluator_3evaluate(PyObject *__pyx_v_
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":100
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":105
  *         del self.scorer
  * 
  *     def evaluate(self, sentence):             # <<<<<<<<<<<<<<
@@ -13976,19 +14111,19 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("evaluate", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":102
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":107
  *     def evaluate(self, sentence):
  *         cdef vector[WordID] hyp
  *         cdef SufficientStats sf = SufficientStats()             # <<<<<<<<<<<<<<
  *         sf.metric = self.metric
  *         sf.stats = new mteval.SufficientStats()
  */
-  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SufficientStats)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 102; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SufficientStats)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 107; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __pyx_v_sf = ((struct __pyx_obj_5_cdec_SufficientStats *)__pyx_t_1);
   __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":103
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":108
  *         cdef vector[WordID] hyp
  *         cdef SufficientStats sf = SufficientStats()
  *         sf.metric = self.metric             # <<<<<<<<<<<<<<
@@ -13997,7 +14132,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5
  */
   __pyx_v_sf->metric = __pyx_v_self->metric;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":104
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":109
  *         cdef SufficientStats sf = SufficientStats()
  *         sf.metric = self.metric
  *         sf.stats = new mteval.SufficientStats()             # <<<<<<<<<<<<<<
@@ -14006,22 +14141,22 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5
  */
   __pyx_v_sf->stats = new SufficientStats();
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":105
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":110
  *         sf.metric = self.metric
  *         sf.stats = new mteval.SufficientStats()
  *         ConvertSentence(string(as_str(sentence.strip())), &hyp)             # <<<<<<<<<<<<<<
  *         self.scorer.get().Evaluate(hyp, sf.stats)
  *         return sf
  */
-  __pyx_t_1 = PyObject_GetAttr(__pyx_v_sentence, __pyx_n_s__strip); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_GetAttr(__pyx_v_sentence, __pyx_n_s__strip); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 105; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
   TD::ConvertSentence(std::string(__pyx_f_5_cdec_as_str(__pyx_t_2, NULL)), (&__pyx_v_hyp));
   __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":106
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":111
  *         sf.stats = new mteval.SufficientStats()
  *         ConvertSentence(string(as_str(sentence.strip())), &hyp)
  *         self.scorer.get().Evaluate(hyp, sf.stats)             # <<<<<<<<<<<<<<
@@ -14030,7 +14165,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx_obj_5
  */
   __pyx_v_self->scorer->get()->Evaluate(__pyx_v_hyp, __pyx_v_sf->stats);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":107
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":112
  *         ConvertSentence(string(as_str(sentence.strip())), &hyp)
  *         self.scorer.get().Evaluate(hyp, sf.stats)
  *         return sf             # <<<<<<<<<<<<<<
@@ -14067,7 +14202,7 @@ static PyObject *__pyx_pw_5_cdec_16SegmentEvaluator_5candidate_set(PyObject *__p
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":109
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":114
  *         return sf
  * 
  *     def candidate_set(self):             # <<<<<<<<<<<<<<
@@ -14085,7 +14220,7 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(struct __pyx_
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("candidate_set", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":110
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":115
  * 
  *     def candidate_set(self):
  *         return CandidateSet(self)             # <<<<<<<<<<<<<<
@@ -14093,12 +14228,12 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(struct __pyx_
  * cdef class Scorer:
  */
   __Pyx_XDECREF(__pyx_r);
-  __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_INCREF(((PyObject *)__pyx_v_self));
   PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_self));
   __Pyx_GIVEREF(((PyObject *)__pyx_v_self));
-  __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_CandidateSet)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 110; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_CandidateSet)), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
   __pyx_r = __pyx_t_2;
@@ -14121,13 +14256,22 @@ static PyObject *__pyx_pf_5_cdec_16SegmentEvaluator_4candidate_set(struct __pyx_
 /* Python wrapper */
 static int __pyx_pw_5_cdec_6Scorer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
 static int __pyx_pw_5_cdec_6Scorer_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
-  char *__pyx_v_name;
+  PyObject *__pyx_v_name = 0;
   static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__name,0};
   int __pyx_r;
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0);
   {
     PyObject* values[1] = {0};
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":121
+ *     cdef mteval.EvaluationMetric* metric
+ * 
+ *     def __cinit__(self, bytes name=None):             # <<<<<<<<<<<<<<
+ *         if name:
+ *             self.name = new string(name)
+ */
+    values[0] = ((PyObject*)Py_None);
     if (unlikely(__pyx_kwds)) {
       Py_ssize_t kw_args;
       const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
@@ -14139,56 +14283,89 @@ static int __pyx_pw_5_cdec_6Scorer_1__cinit__(PyObject *__pyx_v_self, PyObject *
       kw_args = PyDict_Size(__pyx_kwds);
       switch (pos_args) {
         case  0:
-        values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__name);
-        if (likely(values[0])) kw_args--;
-        else goto __pyx_L5_argtuple_error;
+        if (kw_args > 0) {
+          PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s__name);
+          if (value) { values[0] = value; kw_args--; }
+        }
       }
       if (unlikely(kw_args > 0)) {
-        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__cinit__") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
       }
-    } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
-      goto __pyx_L5_argtuple_error;
     } else {
-      values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+      switch (PyTuple_GET_SIZE(__pyx_args)) {
+        case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+        case  0: break;
+        default: goto __pyx_L5_argtuple_error;
+      }
     }
-    __pyx_v_name = PyBytes_AsString(values[0]); if (unlikely((!__pyx_v_name) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    __pyx_v_name = ((PyObject*)values[0]);
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L5_argtuple_error:;
-  __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 115; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __Pyx_RaiseArgtupleInvalid("__cinit__", 0, 0, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   __pyx_L3_error:;
   __Pyx_AddTraceback("_cdec.Scorer.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __Pyx_RefNannyFinishContext();
   return -1;
   __pyx_L4_argument_unpacking_done:;
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_name), (&PyBytes_Type), 1, "name", 1))) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_r = __pyx_pf_5_cdec_6Scorer___cinit__(((struct __pyx_obj_5_cdec_Scorer *)__pyx_v_self), __pyx_v_name);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __pyx_r = -1;
+  __pyx_L0:;
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":115
- *     cdef string* name
- * 
- *     def __cinit__(self, char* name):             # <<<<<<<<<<<<<<
- *         self.name = new string(name)
- * 
- */
-
-static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, char *__pyx_v_name) {
+static int __pyx_pf_5_cdec_6Scorer___cinit__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, PyObject *__pyx_v_name) {
   int __pyx_r;
   __Pyx_RefNannyDeclarations
+  int __pyx_t_1;
+  char *__pyx_t_2;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__cinit__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":116
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":122
+ * 
+ *     def __cinit__(self, bytes name=None):
+ *         if name:             # <<<<<<<<<<<<<<
+ *             self.name = new string(name)
+ *             self.metric = mteval.MetricInstance(self.name[0])
+ */
+  __pyx_t_1 = (((PyObject *)__pyx_v_name) != Py_None) && (PyBytes_GET_SIZE(((PyObject *)__pyx_v_name)) != 0);
+  if (__pyx_t_1) {
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":123
+ *     def __cinit__(self, bytes name=None):
+ *         if name:
+ *             self.name = new string(name)             # <<<<<<<<<<<<<<
+ *             self.metric = mteval.MetricInstance(self.name[0])
  * 
- *     def __cinit__(self, char* name):
- *         self.name = new string(name)             # <<<<<<<<<<<<<<
+ */
+    __pyx_t_2 = PyBytes_AsString(((PyObject *)__pyx_v_name)); if (unlikely((!__pyx_t_2) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 123; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_v_self->name = new std::string(__pyx_t_2);
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":124
+ *         if name:
+ *             self.name = new string(name)
+ *             self.metric = mteval.MetricInstance(self.name[0])             # <<<<<<<<<<<<<<
  * 
  *     def __dealloc__(self):
  */
-  __pyx_v_self->name = new std::string(__pyx_v_name);
+    __pyx_v_self->metric = EvaluationMetric::Instance((__pyx_v_self->name[0]));
+    goto __pyx_L3;
+  }
+  __pyx_L3:;
 
   __pyx_r = 0;
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_AddTraceback("_cdec.Scorer.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = -1;
+  __pyx_L0:;
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
@@ -14202,8 +14379,8 @@ static void __pyx_pw_5_cdec_6Scorer_3__dealloc__(PyObject *__pyx_v_self) {
   __Pyx_RefNannyFinishContext();
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":118
- *         self.name = new string(name)
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":126
+ *             self.metric = mteval.MetricInstance(self.name[0])
  * 
  *     def __dealloc__(self):             # <<<<<<<<<<<<<<
  *         del self.name
@@ -14214,7 +14391,7 @@ static void __pyx_pf_5_cdec_6Scorer_2__dealloc__(CYTHON_UNUSED struct __pyx_obj_
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__dealloc__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":119
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":127
  * 
  *     def __dealloc__(self):
  *         del self.name             # <<<<<<<<<<<<<<
@@ -14252,7 +14429,7 @@ static PyObject *__pyx_pw_5_cdec_6Scorer_5__call__(PyObject *__pyx_v_self, PyObj
         else goto __pyx_L5_argtuple_error;
       }
       if (unlikely(kw_args > 0)) {
-        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__call__") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__call__") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
       }
     } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
       goto __pyx_L5_argtuple_error;
@@ -14263,7 +14440,7 @@ static PyObject *__pyx_pw_5_cdec_6Scorer_5__call__(PyObject *__pyx_v_self, PyObj
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L5_argtuple_error:;
-  __Pyx_RaiseArgtupleInvalid("__call__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 121; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __Pyx_RaiseArgtupleInvalid("__call__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 129; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   __pyx_L3_error:;
   __Pyx_AddTraceback("_cdec.Scorer.__call__", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __Pyx_RefNannyFinishContext();
@@ -14274,16 +14451,15 @@ static PyObject *__pyx_pw_5_cdec_6Scorer_5__call__(PyObject *__pyx_v_self, PyObj
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":121
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":129
  *         del self.name
  * 
  *     def __call__(self, refs):             # <<<<<<<<<<<<<<
- *         cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])
  *         if isinstance(refs, unicode) or isinstance(refs, str):
+ *             refs = [refs]
  */
 
 static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Scorer *__pyx_v_self, PyObject *__pyx_v_refs) {
-  EvaluationMetric *__pyx_v_metric;
   std::vector<std::vector<WordID> > *__pyx_v_refsv;
   std::vector<WordID> *__pyx_v_refv;
   PyObject *__pyx_v_ref = NULL;
@@ -14304,18 +14480,9 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
   __Pyx_RefNannySetupContext("__call__", 0);
   __Pyx_INCREF(__pyx_v_refs);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":122
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":130
  * 
  *     def __call__(self, refs):
- *         cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])             # <<<<<<<<<<<<<<
- *         if isinstance(refs, unicode) or isinstance(refs, str):
- *             refs = [refs]
- */
-  __pyx_v_metric = EvaluationMetric::Instance((__pyx_v_self->name[0]));
-
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":123
- *     def __call__(self, refs):
- *         cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])
  *         if isinstance(refs, unicode) or isinstance(refs, str):             # <<<<<<<<<<<<<<
  *             refs = [refs]
  *         cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]()
@@ -14335,14 +14502,14 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
   }
   if (__pyx_t_4) {
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":124
- *         cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":131
+ *     def __call__(self, refs):
  *         if isinstance(refs, unicode) or isinstance(refs, str):
  *             refs = [refs]             # <<<<<<<<<<<<<<
  *         cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]()
  *         cdef vector[WordID]* refv
  */
-    __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 124; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 131; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_1);
     __Pyx_INCREF(__pyx_v_refs);
     PyList_SET_ITEM(__pyx_t_1, 0, __pyx_v_refs);
@@ -14354,7 +14521,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
   }
   __pyx_L3:;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":125
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":132
  *         if isinstance(refs, unicode) or isinstance(refs, str):
  *             refs = [refs]
  *         cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]()             # <<<<<<<<<<<<<<
@@ -14363,7 +14530,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
  */
   __pyx_v_refsv = new std::vector<std::vector<WordID> >();
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":128
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":135
  *         cdef vector[WordID]* refv
  *         cdef bytes ref_str
  *         for ref in refs:             # <<<<<<<<<<<<<<
@@ -14374,7 +14541,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
     __pyx_t_1 = __pyx_v_refs; __Pyx_INCREF(__pyx_t_1); __pyx_t_5 = 0;
     __pyx_t_6 = NULL;
   } else {
-    __pyx_t_5 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_v_refs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_5 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_v_refs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_1);
     __pyx_t_6 = Py_TYPE(__pyx_t_1)->tp_iternext;
   }
@@ -14390,7 +14557,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
       if (unlikely(!__pyx_t_7)) {
         if (PyErr_Occurred()) {
           if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) PyErr_Clear();
-          else {__pyx_filename = __pyx_f[5]; __pyx_lineno = 128; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          else {__pyx_filename = __pyx_f[5]; __pyx_lineno = 135; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         }
         break;
       }
@@ -14400,7 +14567,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
     __pyx_v_ref = __pyx_t_7;
     __pyx_t_7 = 0;
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":129
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":136
  *         cdef bytes ref_str
  *         for ref in refs:
  *             refv = new vector[WordID]()             # <<<<<<<<<<<<<<
@@ -14409,22 +14576,22 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
  */
     __pyx_v_refv = new std::vector<WordID>();
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":130
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":137
  *         for ref in refs:
  *             refv = new vector[WordID]()
  *             ConvertSentence(string(as_str(ref.strip())), refv)             # <<<<<<<<<<<<<<
  *             refsv.push_back(refv[0])
  *             del refv
  */
-    __pyx_t_7 = PyObject_GetAttr(__pyx_v_ref, __pyx_n_s__strip); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_7 = PyObject_GetAttr(__pyx_v_ref, __pyx_n_s__strip); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_7);
-    __pyx_t_8 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 130; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_8 = PyObject_Call(__pyx_t_7, ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 137; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_8);
     __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
     TD::ConvertSentence(std::string(__pyx_f_5_cdec_as_str(__pyx_t_8, NULL)), __pyx_v_refv);
     __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":131
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":138
  *             refv = new vector[WordID]()
  *             ConvertSentence(string(as_str(ref.strip())), refv)
  *             refsv.push_back(refv[0])             # <<<<<<<<<<<<<<
@@ -14433,7 +14600,7 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
  */
     __pyx_v_refsv->push_back((__pyx_v_refv[0]));
 
-    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":132
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":139
  *             ConvertSentence(string(as_str(ref.strip())), refv)
  *             refsv.push_back(refv[0])
  *             del refv             # <<<<<<<<<<<<<<
@@ -14444,47 +14611,47 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_4__call__(struct __pyx_obj_5_cdec_Score
   }
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":134
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":141
  *             del refv
  *         cdef unsigned i
  *         cdef SegmentEvaluator evaluator = SegmentEvaluator()             # <<<<<<<<<<<<<<
- *         evaluator.metric = metric
- *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+ *         evaluator.metric = self.metric
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](
  */
-  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SegmentEvaluator)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 134; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_SegmentEvaluator)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __pyx_v_evaluator = ((struct __pyx_obj_5_cdec_SegmentEvaluator *)__pyx_t_1);
   __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":135
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":142
  *         cdef unsigned i
  *         cdef SegmentEvaluator evaluator = SegmentEvaluator()
- *         evaluator.metric = metric             # <<<<<<<<<<<<<<
- *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
- *         del refsv # in theory should not delete but store in SegmentEvaluator
+ *         evaluator.metric = self.metric             # <<<<<<<<<<<<<<
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](
+ *                 self.metric.CreateSegmentEvaluator(refsv[0]))
  */
-  __pyx_v_evaluator->metric = __pyx_v_metric;
+  __pyx_v_evaluator->metric = __pyx_v_self->metric;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":136
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":143
  *         cdef SegmentEvaluator evaluator = SegmentEvaluator()
- *         evaluator.metric = metric
- *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))             # <<<<<<<<<<<<<<
+ *         evaluator.metric = self.metric
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](             # <<<<<<<<<<<<<<
+ *                 self.metric.CreateSegmentEvaluator(refsv[0]))
  *         del refsv # in theory should not delete but store in SegmentEvaluator
- *         return evaluator
  */
-  __pyx_v_evaluator->scorer = new boost::shared_ptr<SegmentEvaluator>(__pyx_v_metric->CreateSegmentEvaluator((__pyx_v_refsv[0])));
+  __pyx_v_evaluator->scorer = new boost::shared_ptr<SegmentEvaluator>(__pyx_v_self->metric->CreateSegmentEvaluator((__pyx_v_refsv[0])));
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":137
- *         evaluator.metric = metric
- *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":145
+ *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](
+ *                 self.metric.CreateSegmentEvaluator(refsv[0]))
  *         del refsv # in theory should not delete but store in SegmentEvaluator             # <<<<<<<<<<<<<<
  *         return evaluator
  * 
  */
   delete __pyx_v_refsv;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":138
- *         evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":146
+ *                 self.metric.CreateSegmentEvaluator(refsv[0]))
  *         del refsv # in theory should not delete but store in SegmentEvaluator
  *         return evaluator             # <<<<<<<<<<<<<<
  * 
@@ -14523,7 +14690,7 @@ static PyObject *__pyx_pw_5_cdec_6Scorer_7__str__(PyObject *__pyx_v_self) {
   return __pyx_r;
 }
 
-/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":140
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":148
  *         return evaluator
  * 
  *     def __str__(self):             # <<<<<<<<<<<<<<
@@ -14540,15 +14707,15 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_6__str__(struct __pyx_obj_5_cdec_Scorer
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__str__", 0);
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":141
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":149
  * 
  *     def __str__(self):
  *         return self.name.c_str()             # <<<<<<<<<<<<<<
  * 
- * BLEU = Scorer('IBM_BLEU')
+ * cdef float _compute_score(void* metric_, mteval.SufficientStats* stats):
  */
   __Pyx_XDECREF(__pyx_r);
-  __pyx_t_1 = PyBytes_FromString(__pyx_v_self->name->c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 141; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyBytes_FromString(__pyx_v_self->name->c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 149; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(((PyObject *)__pyx_t_1));
   __pyx_r = ((PyObject *)__pyx_t_1);
   __pyx_t_1 = 0;
@@ -14565,6 +14732,596 @@ static PyObject *__pyx_pf_5_cdec_6Scorer_6__str__(struct __pyx_obj_5_cdec_Scorer
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":151
+ *         return self.name.c_str()
+ * 
+ * cdef float _compute_score(void* metric_, mteval.SufficientStats* stats):             # <<<<<<<<<<<<<<
+ *     cdef Metric metric = <Metric> metric_
+ *     cdef list ss = []
+ */
+
+static float __pyx_f_5_cdec__compute_score(void *__pyx_v_metric_, SufficientStats *__pyx_v_stats) {
+  struct __pyx_obj_5_cdec_Metric *__pyx_v_metric = 0;
+  PyObject *__pyx_v_ss = 0;
+  unsigned int __pyx_v_i;
+  float __pyx_r;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  unsigned int __pyx_t_2;
+  unsigned int __pyx_t_3;
+  int __pyx_t_4;
+  PyObject *__pyx_t_5 = NULL;
+  PyObject *__pyx_t_6 = NULL;
+  float __pyx_t_7;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("_compute_score", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":152
+ * 
+ * cdef float _compute_score(void* metric_, mteval.SufficientStats* stats):
+ *     cdef Metric metric = <Metric> metric_             # <<<<<<<<<<<<<<
+ *     cdef list ss = []
+ *     cdef unsigned i
+ */
+  __Pyx_INCREF(((PyObject *)((struct __pyx_obj_5_cdec_Metric *)__pyx_v_metric_)));
+  __pyx_v_metric = ((struct __pyx_obj_5_cdec_Metric *)__pyx_v_metric_);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":153
+ * cdef float _compute_score(void* metric_, mteval.SufficientStats* stats):
+ *     cdef Metric metric = <Metric> metric_
+ *     cdef list ss = []             # <<<<<<<<<<<<<<
+ *     cdef unsigned i
+ *     for i in range(stats.size()):
+ */
+  __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 153; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_ss = __pyx_t_1;
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":155
+ *     cdef list ss = []
+ *     cdef unsigned i
+ *     for i in range(stats.size()):             # <<<<<<<<<<<<<<
+ *         ss.append(stats[0][i])
+ *     return metric.score(ss)
+ */
+  __pyx_t_2 = __pyx_v_stats->size();
+  for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) {
+    __pyx_v_i = __pyx_t_3;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":156
+ *     cdef unsigned i
+ *     for i in range(stats.size()):
+ *         ss.append(stats[0][i])             # <<<<<<<<<<<<<<
+ *     return metric.score(ss)
+ * 
+ */
+    __pyx_t_1 = PyFloat_FromDouble(((__pyx_v_stats[0])[__pyx_v_i])); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 156; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_1);
+    __pyx_t_4 = PyList_Append(__pyx_v_ss, __pyx_t_1); if (unlikely(__pyx_t_4 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 156; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  }
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":157
+ *     for i in range(stats.size()):
+ *         ss.append(stats[0][i])
+ *     return metric.score(ss)             # <<<<<<<<<<<<<<
+ * 
+ * cdef void _compute_sufficient_stats(void* metric_,
+ */
+  __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_metric), __pyx_n_s__score); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_5);
+  __Pyx_INCREF(((PyObject *)__pyx_v_ss));
+  PyTuple_SET_ITEM(__pyx_t_5, 0, ((PyObject *)__pyx_v_ss));
+  __Pyx_GIVEREF(((PyObject *)__pyx_v_ss));
+  __pyx_t_6 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_5), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_6);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0;
+  __pyx_t_7 = __pyx_PyFloat_AsFloat(__pyx_t_6); if (unlikely((__pyx_t_7 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 157; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
+  __pyx_r = __pyx_t_7;
+  goto __pyx_L0;
+
+  __pyx_r = 0;
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_5);
+  __Pyx_XDECREF(__pyx_t_6);
+  __Pyx_WriteUnraisable("_cdec._compute_score", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = 0;
+  __pyx_L0:;
+  __Pyx_XDECREF((PyObject *)__pyx_v_metric);
+  __Pyx_XDECREF(__pyx_v_ss);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":159
+ *     return metric.score(ss)
+ * 
+ * cdef void _compute_sufficient_stats(void* metric_,             # <<<<<<<<<<<<<<
+ *         string* hyp,
+ *         vector[string]* refs,
+ */
+
+static void __pyx_f_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_, std::string *__pyx_v_hyp, std::vector<std::string> *__pyx_v_refs, SufficientStats *__pyx_v_out) {
+  struct __pyx_obj_5_cdec_Metric *__pyx_v_metric = 0;
+  PyObject *__pyx_v_refs_ = 0;
+  unsigned int __pyx_v_i;
+  PyObject *__pyx_v_ss = 0;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  size_t __pyx_t_2;
+  unsigned int __pyx_t_3;
+  int __pyx_t_4;
+  PyObject *__pyx_t_5 = NULL;
+  PyObject *__pyx_t_6 = NULL;
+  Py_ssize_t __pyx_t_7;
+  float __pyx_t_8;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("_compute_sufficient_stats", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":163
+ *         vector[string]* refs,
+ *         mteval.SufficientStats* out):
+ *     cdef Metric metric = <Metric> metric_             # <<<<<<<<<<<<<<
+ *     cdef list refs_ = []
+ *     cdef unsigned i
+ */
+  __Pyx_INCREF(((PyObject *)((struct __pyx_obj_5_cdec_Metric *)__pyx_v_metric_)));
+  __pyx_v_metric = ((struct __pyx_obj_5_cdec_Metric *)__pyx_v_metric_);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":164
+ *         mteval.SufficientStats* out):
+ *     cdef Metric metric = <Metric> metric_
+ *     cdef list refs_ = []             # <<<<<<<<<<<<<<
+ *     cdef unsigned i
+ *     for i in range(refs.size()):
+ */
+  __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 164; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_v_refs_ = __pyx_t_1;
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":166
+ *     cdef list refs_ = []
+ *     cdef unsigned i
+ *     for i in range(refs.size()):             # <<<<<<<<<<<<<<
+ *         refs_.append(refs[0][i].c_str())
+ *     cdef list ss = metric.evaluate(hyp.c_str(), refs_)
+ */
+  __pyx_t_2 = __pyx_v_refs->size();
+  for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) {
+    __pyx_v_i = __pyx_t_3;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":167
+ *     cdef unsigned i
+ *     for i in range(refs.size()):
+ *         refs_.append(refs[0][i].c_str())             # <<<<<<<<<<<<<<
+ *     cdef list ss = metric.evaluate(hyp.c_str(), refs_)
+ *     out.fields.resize(len(ss))
+ */
+    __pyx_t_1 = PyBytes_FromString(((__pyx_v_refs[0])[__pyx_v_i]).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(((PyObject *)__pyx_t_1));
+    __pyx_t_4 = PyList_Append(__pyx_v_refs_, ((PyObject *)__pyx_t_1)); if (unlikely(__pyx_t_4 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 167; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+  }
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":168
+ *     for i in range(refs.size()):
+ *         refs_.append(refs[0][i].c_str())
+ *     cdef list ss = metric.evaluate(hyp.c_str(), refs_)             # <<<<<<<<<<<<<<
+ *     out.fields.resize(len(ss))
+ *     for i in range(len(ss)):
+ */
+  __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_metric), __pyx_n_s__evaluate); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_5 = PyBytes_FromString(__pyx_v_hyp->c_str()); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(((PyObject *)__pyx_t_5));
+  __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_6);
+  PyTuple_SET_ITEM(__pyx_t_6, 0, ((PyObject *)__pyx_t_5));
+  __Pyx_GIVEREF(((PyObject *)__pyx_t_5));
+  __Pyx_INCREF(((PyObject *)__pyx_v_refs_));
+  PyTuple_SET_ITEM(__pyx_t_6, 1, ((PyObject *)__pyx_v_refs_));
+  __Pyx_GIVEREF(((PyObject *)__pyx_v_refs_));
+  __pyx_t_5 = 0;
+  __pyx_t_5 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_5);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0;
+  if (!(likely(PyList_CheckExact(__pyx_t_5))||((__pyx_t_5) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected list, got %.200s", Py_TYPE(__pyx_t_5)->tp_name), 0))) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 168; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_v_ss = ((PyObject*)__pyx_t_5);
+  __pyx_t_5 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":169
+ *         refs_.append(refs[0][i].c_str())
+ *     cdef list ss = metric.evaluate(hyp.c_str(), refs_)
+ *     out.fields.resize(len(ss))             # <<<<<<<<<<<<<<
+ *     for i in range(len(ss)):
+ *         out.fields[i] = ss[i]
+ */
+  if (unlikely(((PyObject *)__pyx_v_ss) == Py_None)) {
+    PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 169; __pyx_clineno = __LINE__; goto __pyx_L1_error;} 
+  }
+  __pyx_t_7 = PyList_GET_SIZE(((PyObject *)__pyx_v_ss)); 
+  __pyx_v_out->fields.resize(__pyx_t_7);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":170
+ *     cdef list ss = metric.evaluate(hyp.c_str(), refs_)
+ *     out.fields.resize(len(ss))
+ *     for i in range(len(ss)):             # <<<<<<<<<<<<<<
+ *         out.fields[i] = ss[i]
+ * 
+ */
+  if (unlikely(((PyObject *)__pyx_v_ss) == Py_None)) {
+    PyErr_SetString(PyExc_TypeError, "object of type 'NoneType' has no len()"); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 170; __pyx_clineno = __LINE__; goto __pyx_L1_error;} 
+  }
+  __pyx_t_7 = PyList_GET_SIZE(((PyObject *)__pyx_v_ss)); 
+  for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_7; __pyx_t_3+=1) {
+    __pyx_v_i = __pyx_t_3;
+
+    /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":171
+ *     out.fields.resize(len(ss))
+ *     for i in range(len(ss)):
+ *         out.fields[i] = ss[i]             # <<<<<<<<<<<<<<
+ * 
+ * cdef class Metric:
+ */
+    __pyx_t_5 = __Pyx_GetItemInt_List(((PyObject *)__pyx_v_ss), __pyx_v_i, sizeof(unsigned int)+1, PyLong_FromUnsignedLong); if (!__pyx_t_5) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_GOTREF(__pyx_t_5);
+    __pyx_t_8 = __pyx_PyFloat_AsFloat(__pyx_t_5); if (unlikely((__pyx_t_8 == (float)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 171; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
+    (__pyx_v_out->fields[__pyx_v_i]) = __pyx_t_8;
+  }
+
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_5);
+  __Pyx_XDECREF(__pyx_t_6);
+  __Pyx_WriteUnraisable("_cdec._compute_sufficient_stats", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_L0:;
+  __Pyx_XDECREF((PyObject *)__pyx_v_metric);
+  __Pyx_XDECREF(__pyx_v_refs_);
+  __Pyx_XDECREF(__pyx_v_ss);
+  __Pyx_RefNannyFinishContext();
+}
+
+/* Python wrapper */
+static int __pyx_pw_5_cdec_6Metric_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static int __pyx_pw_5_cdec_6Metric_1__cinit__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  int __pyx_r;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__cinit__ (wrapper)", 0);
+  if (unlikely(PyTuple_GET_SIZE(__pyx_args) > 0)) {
+    __Pyx_RaiseArgtupleInvalid("__cinit__", 1, 0, 0, PyTuple_GET_SIZE(__pyx_args)); return -1;}
+  if (unlikely(__pyx_kwds) && unlikely(PyDict_Size(__pyx_kwds) > 0) && unlikely(!__Pyx_CheckKeywordStrings(__pyx_kwds, "__cinit__", 0))) return -1;
+  __pyx_r = __pyx_pf_5_cdec_6Metric___cinit__(((struct __pyx_obj_5_cdec_Metric *)__pyx_v_self));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":175
+ * cdef class Metric:
+ *     cdef Scorer scorer
+ *     def __cinit__(self):             # <<<<<<<<<<<<<<
+ *         self.scorer = Scorer()
+ *         self.scorer.name = new string(as_str(self.__class__.__name__))
+ */
+
+static int __pyx_pf_5_cdec_6Metric___cinit__(struct __pyx_obj_5_cdec_Metric *__pyx_v_self) {
+  int __pyx_r;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__cinit__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":176
+ *     cdef Scorer scorer
+ *     def __cinit__(self):
+ *         self.scorer = Scorer()             # <<<<<<<<<<<<<<
+ *         self.scorer.name = new string(as_str(self.__class__.__name__))
+ *         self.scorer.metric = mteval.PyMetricInstance(self.scorer.name[0],
+ */
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Scorer)), ((PyObject *)__pyx_empty_tuple), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 176; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __Pyx_GIVEREF(__pyx_t_1);
+  __Pyx_GOTREF(__pyx_v_self->scorer);
+  __Pyx_DECREF(((PyObject *)__pyx_v_self->scorer));
+  __pyx_v_self->scorer = ((struct __pyx_obj_5_cdec_Scorer *)__pyx_t_1);
+  __pyx_t_1 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":177
+ *     def __cinit__(self):
+ *         self.scorer = Scorer()
+ *         self.scorer.name = new string(as_str(self.__class__.__name__))             # <<<<<<<<<<<<<<
+ *         self.scorer.metric = mteval.PyMetricInstance(self.scorer.name[0],
+ *                 <void*> self, _compute_sufficient_stats, _compute_score)
+ */
+  __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s____class__); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 177; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s____name__); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 177; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __pyx_v_self->scorer->name = new std::string(__pyx_f_5_cdec_as_str(__pyx_t_2, NULL));
+  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":178
+ *         self.scorer = Scorer()
+ *         self.scorer.name = new string(as_str(self.__class__.__name__))
+ *         self.scorer.metric = mteval.PyMetricInstance(self.scorer.name[0],             # <<<<<<<<<<<<<<
+ *                 <void*> self, _compute_sufficient_stats, _compute_score)
+ * 
+ */
+  __pyx_v_self->scorer->metric = PythonEvaluationMetric::Instance((__pyx_v_self->scorer->name[0]), ((void *)__pyx_v_self), __pyx_f_5_cdec__compute_sufficient_stats, __pyx_f_5_cdec__compute_score);
+
+  __pyx_r = 0;
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_AddTraceback("_cdec.Metric.__cinit__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = -1;
+  __pyx_L0:;
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_6Metric_3__call__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyObject *__pyx_pw_5_cdec_6Metric_3__call__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  PyObject *__pyx_v_refs = 0;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__refs,0};
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("__call__ (wrapper)", 0);
+  {
+    PyObject* values[1] = {0};
+    if (unlikely(__pyx_kwds)) {
+      Py_ssize_t kw_args;
+      const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
+      switch (pos_args) {
+        case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+        case  0: break;
+        default: goto __pyx_L5_argtuple_error;
+      }
+      kw_args = PyDict_Size(__pyx_kwds);
+      switch (pos_args) {
+        case  0:
+        values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs);
+        if (likely(values[0])) kw_args--;
+        else goto __pyx_L5_argtuple_error;
+      }
+      if (unlikely(kw_args > 0)) {
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__call__") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
+      goto __pyx_L5_argtuple_error;
+    } else {
+      values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+    }
+    __pyx_v_refs = values[0];
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("__call__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 181; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("_cdec.Metric.__call__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_r = __pyx_pf_5_cdec_6Metric_2__call__(((struct __pyx_obj_5_cdec_Metric *)__pyx_v_self), __pyx_v_refs);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":181
+ *                 <void*> self, _compute_sufficient_stats, _compute_score)
+ * 
+ *     def __call__(self, refs):             # <<<<<<<<<<<<<<
+ *         return self.scorer(refs)
+ * 
+ */
+
+static PyObject *__pyx_pf_5_cdec_6Metric_2__call__(struct __pyx_obj_5_cdec_Metric *__pyx_v_self, PyObject *__pyx_v_refs) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  PyObject *__pyx_t_2 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("__call__", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":182
+ * 
+ *     def __call__(self, refs):
+ *         return self.scorer(refs)             # <<<<<<<<<<<<<<
+ * 
+ *     def score(SufficientStats stats):
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 182; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __Pyx_INCREF(__pyx_v_refs);
+  PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_refs);
+  __Pyx_GIVEREF(__pyx_v_refs);
+  __pyx_t_2 = PyObject_Call(((PyObject *)__pyx_v_self->scorer), ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 182; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+  __pyx_r = __pyx_t_2;
+  __pyx_t_2 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_XDECREF(__pyx_t_2);
+  __Pyx_AddTraceback("_cdec.Metric.__call__", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_6Metric_5score(PyObject *__pyx_v_stats, CYTHON_UNUSED PyObject *unused); /*proto*/
+static PyObject *__pyx_pw_5_cdec_6Metric_5score(PyObject *__pyx_v_stats, CYTHON_UNUSED PyObject *unused) {
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("score (wrapper)", 0);
+  __pyx_r = __pyx_pf_5_cdec_6Metric_4score(((struct __pyx_obj_5_cdec_Metric *)__pyx_v_stats));
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":184
+ *         return self.scorer(refs)
+ * 
+ *     def score(SufficientStats stats):             # <<<<<<<<<<<<<<
+ *         return 0
+ * 
+ */
+
+static PyObject *__pyx_pf_5_cdec_6Metric_4score(CYTHON_UNUSED struct __pyx_obj_5_cdec_Metric *__pyx_v_stats) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("score", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":185
+ * 
+ *     def score(SufficientStats stats):
+ *         return 0             # <<<<<<<<<<<<<<
+ * 
+ *     def evaluate(self, hyp, refs):
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __Pyx_INCREF(__pyx_int_0);
+  __pyx_r = __pyx_int_0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* Python wrapper */
+static PyObject *__pyx_pw_5_cdec_6Metric_7evaluate(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
+static PyObject *__pyx_pw_5_cdec_6Metric_7evaluate(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) {
+  CYTHON_UNUSED PyObject *__pyx_v_hyp = 0;
+  CYTHON_UNUSED PyObject *__pyx_v_refs = 0;
+  static PyObject **__pyx_pyargnames[] = {&__pyx_n_s__hyp,&__pyx_n_s__refs,0};
+  PyObject *__pyx_r = 0;
+  __Pyx_RefNannyDeclarations
+  __Pyx_RefNannySetupContext("evaluate (wrapper)", 0);
+  {
+    PyObject* values[2] = {0,0};
+    if (unlikely(__pyx_kwds)) {
+      Py_ssize_t kw_args;
+      const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args);
+      switch (pos_args) {
+        case  2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+        case  1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+        case  0: break;
+        default: goto __pyx_L5_argtuple_error;
+      }
+      kw_args = PyDict_Size(__pyx_kwds);
+      switch (pos_args) {
+        case  0:
+        values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__hyp);
+        if (likely(values[0])) kw_args--;
+        else goto __pyx_L5_argtuple_error;
+        case  1:
+        values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s__refs);
+        if (likely(values[1])) kw_args--;
+        else {
+          __Pyx_RaiseArgtupleInvalid("evaluate", 1, 2, 2, 1); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+        }
+      }
+      if (unlikely(kw_args > 0)) {
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "evaluate") < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+      }
+    } else if (PyTuple_GET_SIZE(__pyx_args) != 2) {
+      goto __pyx_L5_argtuple_error;
+    } else {
+      values[0] = PyTuple_GET_ITEM(__pyx_args, 0);
+      values[1] = PyTuple_GET_ITEM(__pyx_args, 1);
+    }
+    __pyx_v_hyp = values[0];
+    __pyx_v_refs = values[1];
+  }
+  goto __pyx_L4_argument_unpacking_done;
+  __pyx_L5_argtuple_error:;
+  __Pyx_RaiseArgtupleInvalid("evaluate", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[5]; __pyx_lineno = 187; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __pyx_L3_error:;
+  __Pyx_AddTraceback("_cdec.Metric.evaluate", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __Pyx_RefNannyFinishContext();
+  return NULL;
+  __pyx_L4_argument_unpacking_done:;
+  __pyx_r = __pyx_pf_5_cdec_6Metric_6evaluate(((struct __pyx_obj_5_cdec_Metric *)__pyx_v_self), __pyx_v_hyp, __pyx_v_refs);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
+
+/* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":187
+ *         return 0
+ * 
+ *     def evaluate(self, hyp, refs):             # <<<<<<<<<<<<<<
+ *         return []
+ * 
+ */
+
+static PyObject *__pyx_pf_5_cdec_6Metric_6evaluate(CYTHON_UNUSED struct __pyx_obj_5_cdec_Metric *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v_hyp, CYTHON_UNUSED PyObject *__pyx_v_refs) {
+  PyObject *__pyx_r = NULL;
+  __Pyx_RefNannyDeclarations
+  PyObject *__pyx_t_1 = NULL;
+  int __pyx_lineno = 0;
+  const char *__pyx_filename = NULL;
+  int __pyx_clineno = 0;
+  __Pyx_RefNannySetupContext("evaluate", 0);
+
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":188
+ * 
+ *     def evaluate(self, hyp, refs):
+ *         return []             # <<<<<<<<<<<<<<
+ * 
+ * BLEU = Scorer('IBM_BLEU')
+ */
+  __Pyx_XDECREF(__pyx_r);
+  __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 188; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  __pyx_r = ((PyObject *)__pyx_t_1);
+  __pyx_t_1 = 0;
+  goto __pyx_L0;
+
+  __pyx_r = Py_None; __Pyx_INCREF(Py_None);
+  goto __pyx_L0;
+  __pyx_L1_error:;
+  __Pyx_XDECREF(__pyx_t_1);
+  __Pyx_AddTraceback("_cdec.Metric.evaluate", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __pyx_r = NULL;
+  __pyx_L0:;
+  __Pyx_XGIVEREF(__pyx_r);
+  __Pyx_RefNannyFinishContext();
+  return __pyx_r;
+}
 static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
 
 /* Python wrapper */
@@ -14866,7 +15623,7 @@ static PyObject *__pyx_gb_5_cdec_4generator16(__pyx_GeneratorObject *__pyx_gener
         __Pyx_INCREF(__pyx_cur_scope->__pyx_v_info);
         PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_cur_scope->__pyx_v_info);
         __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_info);
-        __pyx_t_7 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_43), ((PyObject *)__pyx_t_6)); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_7 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_45), ((PyObject *)__pyx_t_6)); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(((PyObject *)__pyx_t_7));
         __Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0;
         __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
@@ -15245,7 +16002,7 @@ static PyObject *__pyx_gb_5_cdec_7Decoder_9__cinit___2generator20(__pyx_Generato
     __Pyx_GIVEREF(__pyx_t_3);
     __pyx_cur_scope->__pyx_v_kv = __pyx_t_3;
     __pyx_t_3 = 0;
-    __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_44), __pyx_cur_scope->__pyx_v_kv); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_46), __pyx_cur_scope->__pyx_v_kv); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(((PyObject *)__pyx_t_3));
     __pyx_r = ((PyObject *)__pyx_t_3);
     __pyx_t_3 = 0;
@@ -15336,7 +16093,7 @@ static int __pyx_pf_5_cdec_7Decoder___cinit__(struct __pyx_obj_5_cdec_Decoder *_
  */
     __pyx_t_2 = PyObject_GetAttr(__pyx_cur_scope->__pyx_v_config, __pyx_n_s__get); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_2);
-    __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_k_tuple_45), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_k_tuple_47), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
     __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
     __pyx_v_formalism = __pyx_t_3;
@@ -15401,7 +16158,7 @@ static int __pyx_pf_5_cdec_7Decoder___cinit__(struct __pyx_obj_5_cdec_Decoder *_
  */
       __pyx_t_3 = __Pyx_GetName(__pyx_m, __pyx_n_s__InvalidConfig); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_3);
-      __pyx_t_2 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_46), __pyx_v_formalism); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_2 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_48), __pyx_v_formalism); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(((PyObject *)__pyx_t_2));
       __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_6);
@@ -15807,7 +16564,7 @@ static int __pyx_pf_5_cdec_7Decoder_7weights_2__set__(struct __pyx_obj_5_cdec_De
  * 
  *     property formalism:
  */
-    __pyx_t_1 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_47), ((PyObject *)Py_TYPE(__pyx_v_weights))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_1 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_49), ((PyObject *)Py_TYPE(__pyx_v_weights))); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(((PyObject *)__pyx_t_1));
     __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
@@ -16038,7 +16795,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_
             __pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__startswith); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L7_error;}
             __Pyx_GOTREF(__pyx_t_2);
             __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-            __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_k_tuple_49), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L7_error;}
+            __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_k_tuple_51), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L7_error;}
             __Pyx_GOTREF(__pyx_t_1);
             __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
             __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_10 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L7_error;}
@@ -16207,7 +16964,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_4read_weights(struct __pyx_obj_5_cdec_
     }
     /*finally:*/ {
       if (__pyx_t_3) {
-        __pyx_t_7 = PyObject_Call(__pyx_t_3, __pyx_k_tuple_50, NULL);
+        __pyx_t_7 = PyObject_Call(__pyx_t_3, __pyx_k_tuple_52, NULL);
         __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
         if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_7);
@@ -16412,7 +17169,7 @@ static PyObject *__pyx_pf_5_cdec_7Decoder_6translate(struct __pyx_obj_5_cdec_Dec
  *         if grammar:
  *             if isinstance(grammar, str) or isinstance(grammar, unicode):
  */
-    __pyx_t_5 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_51), ((PyObject *)Py_TYPE(__pyx_v_sentence))); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_5 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_53), ((PyObject *)Py_TYPE(__pyx_v_sentence))); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(((PyObject *)__pyx_t_5));
     __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_1);
@@ -19213,6 +19970,13 @@ static void __pyx_tp_dealloc_5_cdec_SufficientStats(PyObject *o) {
   }
   (*Py_TYPE(o)->tp_free)(o);
 }
+static PyObject *__pyx_sq_item_5_cdec_SufficientStats(PyObject *o, Py_ssize_t i) {
+  PyObject *r;
+  PyObject *x = PyInt_FromSsize_t(i); if(!x) return 0;
+  r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x);
+  Py_DECREF(x);
+  return r;
+}
 
 static PyObject *__pyx_getprop_5_cdec_15SufficientStats_score(PyObject *o, void *x) {
   return __pyx_pw_5_cdec_15SufficientStats_5score_1__get__(o);
@@ -19233,7 +19997,7 @@ static struct PyGetSetDef __pyx_getsets_5_cdec_SufficientStats[] = {
 };
 
 static PyNumberMethods __pyx_tp_as_number_SufficientStats = {
-  __pyx_pw_5_cdec_15SufficientStats_10__add__, /*nb_add*/
+  __pyx_pw_5_cdec_15SufficientStats_12__add__, /*nb_add*/
   0, /*nb_subtract*/
   0, /*nb_multiply*/
   #if PY_MAJOR_VERSION < 3
@@ -19268,7 +20032,7 @@ static PyNumberMethods __pyx_tp_as_number_SufficientStats = {
   #if PY_MAJOR_VERSION < 3
   0, /*nb_hex*/
   #endif
-  __pyx_pw_5_cdec_15SufficientStats_8__iadd__, /*nb_inplace_add*/
+  __pyx_pw_5_cdec_15SufficientStats_10__iadd__, /*nb_inplace_add*/
   0, /*nb_inplace_subtract*/
   0, /*nb_inplace_multiply*/
   #if PY_MAJOR_VERSION < 3
@@ -19294,7 +20058,7 @@ static PySequenceMethods __pyx_tp_as_sequence_SufficientStats = {
   __pyx_pw_5_cdec_15SufficientStats_3__len__, /*sq_length*/
   0, /*sq_concat*/
   0, /*sq_repeat*/
-  0, /*sq_item*/
+  __pyx_sq_item_5_cdec_SufficientStats, /*sq_item*/
   0, /*sq_slice*/
   0, /*sq_ass_item*/
   0, /*sq_ass_slice*/
@@ -19305,7 +20069,7 @@ static PySequenceMethods __pyx_tp_as_sequence_SufficientStats = {
 
 static PyMappingMethods __pyx_tp_as_mapping_SufficientStats = {
   __pyx_pw_5_cdec_15SufficientStats_3__len__, /*mp_length*/
-  0, /*mp_subscript*/
+  __pyx_pw_5_cdec_15SufficientStats_8__getitem__, /*mp_subscript*/
   0, /*mp_ass_subscript*/
 };
 
@@ -19933,6 +20697,202 @@ static PyTypeObject __pyx_type_5_cdec_Scorer = {
   #endif
 };
 
+static PyObject *__pyx_tp_new_5_cdec_Metric(PyTypeObject *t, PyObject *a, PyObject *k) {
+  struct __pyx_obj_5_cdec_Metric *p;
+  PyObject *o = (*t->tp_alloc)(t, 0);
+  if (!o) return 0;
+  p = ((struct __pyx_obj_5_cdec_Metric *)o);
+  p->scorer = ((struct __pyx_obj_5_cdec_Scorer *)Py_None); Py_INCREF(Py_None);
+  if (__pyx_pw_5_cdec_6Metric_1__cinit__(o, __pyx_empty_tuple, NULL) < 0) {
+    Py_DECREF(o); o = 0;
+  }
+  return o;
+}
+
+static void __pyx_tp_dealloc_5_cdec_Metric(PyObject *o) {
+  struct __pyx_obj_5_cdec_Metric *p = (struct __pyx_obj_5_cdec_Metric *)o;
+  Py_XDECREF(((PyObject *)p->scorer));
+  (*Py_TYPE(o)->tp_free)(o);
+}
+
+static int __pyx_tp_traverse_5_cdec_Metric(PyObject *o, visitproc v, void *a) {
+  int e;
+  struct __pyx_obj_5_cdec_Metric *p = (struct __pyx_obj_5_cdec_Metric *)o;
+  if (p->scorer) {
+    e = (*v)(((PyObject*)p->scorer), a); if (e) return e;
+  }
+  return 0;
+}
+
+static int __pyx_tp_clear_5_cdec_Metric(PyObject *o) {
+  struct __pyx_obj_5_cdec_Metric *p = (struct __pyx_obj_5_cdec_Metric *)o;
+  PyObject* tmp;
+  tmp = ((PyObject*)p->scorer);
+  p->scorer = ((struct __pyx_obj_5_cdec_Scorer *)Py_None); Py_INCREF(Py_None);
+  Py_XDECREF(tmp);
+  return 0;
+}
+
+static PyMethodDef __pyx_methods_5_cdec_Metric[] = {
+  {__Pyx_NAMESTR("score"), (PyCFunction)__pyx_pw_5_cdec_6Metric_5score, METH_NOARGS, __Pyx_DOCSTR(0)},
+  {__Pyx_NAMESTR("evaluate"), (PyCFunction)__pyx_pw_5_cdec_6Metric_7evaluate, METH_VARARGS|METH_KEYWORDS, __Pyx_DOCSTR(0)},
+  {0, 0, 0, 0}
+};
+
+static PyNumberMethods __pyx_tp_as_number_Metric = {
+  0, /*nb_add*/
+  0, /*nb_subtract*/
+  0, /*nb_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_divide*/
+  #endif
+  0, /*nb_remainder*/
+  0, /*nb_divmod*/
+  0, /*nb_power*/
+  0, /*nb_negative*/
+  0, /*nb_positive*/
+  0, /*nb_absolute*/
+  0, /*nb_nonzero*/
+  0, /*nb_invert*/
+  0, /*nb_lshift*/
+  0, /*nb_rshift*/
+  0, /*nb_and*/
+  0, /*nb_xor*/
+  0, /*nb_or*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_coerce*/
+  #endif
+  0, /*nb_int*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_long*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*nb_float*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_oct*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_hex*/
+  #endif
+  0, /*nb_inplace_add*/
+  0, /*nb_inplace_subtract*/
+  0, /*nb_inplace_multiply*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*nb_inplace_divide*/
+  #endif
+  0, /*nb_inplace_remainder*/
+  0, /*nb_inplace_power*/
+  0, /*nb_inplace_lshift*/
+  0, /*nb_inplace_rshift*/
+  0, /*nb_inplace_and*/
+  0, /*nb_inplace_xor*/
+  0, /*nb_inplace_or*/
+  0, /*nb_floor_divide*/
+  0, /*nb_true_divide*/
+  0, /*nb_inplace_floor_divide*/
+  0, /*nb_inplace_true_divide*/
+  #if PY_VERSION_HEX >= 0x02050000
+  0, /*nb_index*/
+  #endif
+};
+
+static PySequenceMethods __pyx_tp_as_sequence_Metric = {
+  0, /*sq_length*/
+  0, /*sq_concat*/
+  0, /*sq_repeat*/
+  0, /*sq_item*/
+  0, /*sq_slice*/
+  0, /*sq_ass_item*/
+  0, /*sq_ass_slice*/
+  0, /*sq_contains*/
+  0, /*sq_inplace_concat*/
+  0, /*sq_inplace_repeat*/
+};
+
+static PyMappingMethods __pyx_tp_as_mapping_Metric = {
+  0, /*mp_length*/
+  0, /*mp_subscript*/
+  0, /*mp_ass_subscript*/
+};
+
+static PyBufferProcs __pyx_tp_as_buffer_Metric = {
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getreadbuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getwritebuffer*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getsegcount*/
+  #endif
+  #if PY_MAJOR_VERSION < 3
+  0, /*bf_getcharbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_getbuffer*/
+  #endif
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*bf_releasebuffer*/
+  #endif
+};
+
+static PyTypeObject __pyx_type_5_cdec_Metric = {
+  PyVarObject_HEAD_INIT(0, 0)
+  __Pyx_NAMESTR("_cdec.Metric"), /*tp_name*/
+  sizeof(struct __pyx_obj_5_cdec_Metric), /*tp_basicsize*/
+  0, /*tp_itemsize*/
+  __pyx_tp_dealloc_5_cdec_Metric, /*tp_dealloc*/
+  0, /*tp_print*/
+  0, /*tp_getattr*/
+  0, /*tp_setattr*/
+  #if PY_MAJOR_VERSION < 3
+  0, /*tp_compare*/
+  #else
+  0, /*reserved*/
+  #endif
+  0, /*tp_repr*/
+  &__pyx_tp_as_number_Metric, /*tp_as_number*/
+  &__pyx_tp_as_sequence_Metric, /*tp_as_sequence*/
+  &__pyx_tp_as_mapping_Metric, /*tp_as_mapping*/
+  0, /*tp_hash*/
+  __pyx_pw_5_cdec_6Metric_3__call__, /*tp_call*/
+  0, /*tp_str*/
+  0, /*tp_getattro*/
+  0, /*tp_setattro*/
+  &__pyx_tp_as_buffer_Metric, /*tp_as_buffer*/
+  Py_TPFLAGS_DEFAULT|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/
+  0, /*tp_doc*/
+  __pyx_tp_traverse_5_cdec_Metric, /*tp_traverse*/
+  __pyx_tp_clear_5_cdec_Metric, /*tp_clear*/
+  0, /*tp_richcompare*/
+  0, /*tp_weaklistoffset*/
+  0, /*tp_iter*/
+  0, /*tp_iternext*/
+  __pyx_methods_5_cdec_Metric, /*tp_methods*/
+  0, /*tp_members*/
+  0, /*tp_getset*/
+  0, /*tp_base*/
+  0, /*tp_dict*/
+  0, /*tp_descr_get*/
+  0, /*tp_descr_set*/
+  0, /*tp_dictoffset*/
+  0, /*tp_init*/
+  0, /*tp_alloc*/
+  __pyx_tp_new_5_cdec_Metric, /*tp_new*/
+  0, /*tp_free*/
+  0, /*tp_is_gc*/
+  0, /*tp_bases*/
+  0, /*tp_mro*/
+  0, /*tp_cache*/
+  0, /*tp_subclasses*/
+  0, /*tp_weaklist*/
+  0, /*tp_del*/
+  #if PY_VERSION_HEX >= 0x02060000
+  0, /*tp_version_tag*/
+  #endif
+};
+
 static PyObject *__pyx_tp_new_5_cdec_Decoder(PyTypeObject *t, PyObject *a, PyObject *k) {
   struct __pyx_obj_5_cdec_Decoder *p;
   PyObject *o = (*t->tp_alloc)(t, 0);
@@ -25197,13 +26157,14 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_kp_s_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 1, 0},
   {&__pyx_kp_s_41, __pyx_k_41, sizeof(__pyx_k_41), 0, 0, 1, 0},
   {&__pyx_kp_s_43, __pyx_k_43, sizeof(__pyx_k_43), 0, 0, 1, 0},
-  {&__pyx_kp_s_44, __pyx_k_44, sizeof(__pyx_k_44), 0, 0, 1, 0},
+  {&__pyx_kp_s_45, __pyx_k_45, sizeof(__pyx_k_45), 0, 0, 1, 0},
   {&__pyx_kp_s_46, __pyx_k_46, sizeof(__pyx_k_46), 0, 0, 1, 0},
-  {&__pyx_kp_s_47, __pyx_k_47, sizeof(__pyx_k_47), 0, 0, 1, 0},
   {&__pyx_kp_s_48, __pyx_k_48, sizeof(__pyx_k_48), 0, 0, 1, 0},
-  {&__pyx_kp_s_51, __pyx_k_51, sizeof(__pyx_k_51), 0, 0, 1, 0},
-  {&__pyx_kp_s_54, __pyx_k_54, sizeof(__pyx_k_54), 0, 0, 1, 0},
-  {&__pyx_kp_s_59, __pyx_k_59, sizeof(__pyx_k_59), 0, 0, 1, 0},
+  {&__pyx_kp_s_49, __pyx_k_49, sizeof(__pyx_k_49), 0, 0, 1, 0},
+  {&__pyx_kp_s_50, __pyx_k_50, sizeof(__pyx_k_50), 0, 0, 1, 0},
+  {&__pyx_kp_s_53, __pyx_k_53, sizeof(__pyx_k_53), 0, 0, 1, 0},
+  {&__pyx_kp_s_56, __pyx_k_56, sizeof(__pyx_k_56), 0, 0, 1, 0},
+  {&__pyx_kp_s_61, __pyx_k_61, sizeof(__pyx_k_61), 0, 0, 1, 0},
   {&__pyx_kp_s_7, __pyx_k_7, sizeof(__pyx_k_7), 0, 0, 1, 0},
   {&__pyx_kp_s_8, __pyx_k_8, sizeof(__pyx_k_8), 0, 0, 1, 0},
   {&__pyx_kp_s_9, __pyx_k_9, sizeof(__pyx_k_9), 0, 0, 1, 0},
@@ -25218,9 +26179,11 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_n_s__TER, __pyx_k__TER, sizeof(__pyx_k__TER), 0, 0, 1, 1},
   {&__pyx_n_s__TypeError, __pyx_k__TypeError, sizeof(__pyx_k__TypeError), 0, 0, 1, 1},
   {&__pyx_n_s__ValueError, __pyx_k__ValueError, sizeof(__pyx_k__ValueError), 0, 0, 1, 1},
+  {&__pyx_n_s____class__, __pyx_k____class__, sizeof(__pyx_k____class__), 0, 0, 1, 1},
   {&__pyx_n_s____enter__, __pyx_k____enter__, sizeof(__pyx_k____enter__), 0, 0, 1, 1},
   {&__pyx_n_s____exit__, __pyx_k____exit__, sizeof(__pyx_k____exit__), 0, 0, 1, 1},
   {&__pyx_n_s____main__, __pyx_k____main__, sizeof(__pyx_k____main__), 0, 0, 1, 1},
+  {&__pyx_n_s____name__, __pyx_k____name__, sizeof(__pyx_k____name__), 0, 0, 1, 1},
   {&__pyx_n_s____test__, __pyx_k____test__, sizeof(__pyx_k____test__), 0, 0, 1, 1},
   {&__pyx_n_s___cdec, __pyx_k___cdec, sizeof(__pyx_k___cdec), 0, 0, 1, 1},
   {&__pyx_n_s___make_config, __pyx_k___make_config, sizeof(__pyx_k___make_config), 0, 0, 1, 1},
@@ -25239,6 +26202,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_n_s__encoding, __pyx_k__encoding, sizeof(__pyx_k__encoding), 0, 0, 1, 1},
   {&__pyx_n_s__enumerate, __pyx_k__enumerate, sizeof(__pyx_k__enumerate), 0, 0, 1, 1},
   {&__pyx_n_s__eval, __pyx_k__eval, sizeof(__pyx_k__eval), 0, 0, 1, 1},
+  {&__pyx_n_s__evaluate, __pyx_k__evaluate, sizeof(__pyx_k__evaluate), 0, 0, 1, 1},
   {&__pyx_n_s__evaluator, __pyx_k__evaluator, sizeof(__pyx_k__evaluator), 0, 0, 1, 1},
   {&__pyx_n_s__f, __pyx_k__f, sizeof(__pyx_k__f), 0, 0, 1, 1},
   {&__pyx_n_s__formalism, __pyx_k__formalism, sizeof(__pyx_k__formalism), 0, 0, 1, 1},
@@ -25246,6 +26210,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_n_s__genexpr, __pyx_k__genexpr, sizeof(__pyx_k__genexpr), 0, 0, 1, 1},
   {&__pyx_n_s__get, __pyx_k__get, sizeof(__pyx_k__get), 0, 0, 1, 1},
   {&__pyx_n_s__grammar, __pyx_k__grammar, sizeof(__pyx_k__grammar), 0, 0, 1, 1},
+  {&__pyx_n_s__hyp, __pyx_k__hyp, sizeof(__pyx_k__hyp), 0, 0, 1, 1},
   {&__pyx_n_s__hypergraph, __pyx_k__hypergraph, sizeof(__pyx_k__hypergraph), 0, 0, 1, 1},
   {&__pyx_n_s__i, __pyx_k__i, sizeof(__pyx_k__i), 0, 0, 1, 1},
   {&__pyx_n_s__in_edges, __pyx_k__in_edges, sizeof(__pyx_k__in_edges), 0, 0, 1, 1},
@@ -25271,6 +26236,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_n_s__replace, __pyx_k__replace, sizeof(__pyx_k__replace), 0, 0, 1, 1},
   {&__pyx_n_s__rules, __pyx_k__rules, sizeof(__pyx_k__rules), 0, 0, 1, 1},
   {&__pyx_n_s__scfg, __pyx_k__scfg, sizeof(__pyx_k__scfg), 0, 0, 1, 1},
+  {&__pyx_n_s__score, __pyx_k__score, sizeof(__pyx_k__score), 0, 0, 1, 1},
   {&__pyx_n_s__scores, __pyx_k__scores, sizeof(__pyx_k__scores), 0, 0, 1, 1},
   {&__pyx_n_s__self, __pyx_k__self, sizeof(__pyx_k__self), 0, 0, 1, 1},
   {&__pyx_n_s__sentence, __pyx_k__sentence, sizeof(__pyx_k__sentence), 0, 0, 1, 1},
@@ -25350,7 +26316,7 @@ static int __Pyx_InitCachedConstants(void) {
  *         for trule in rules:
  *             if not isinstance(trule, BaseTRule):
  *                 raise ValueError('the grammar should contain TRule objects')             # <<<<<<<<<<<<<<
- *             _g.AddRule((<TRule> trule).rule[0])
+ *             _g.AddRule((<BaseTRule> trule).rule[0])
  */
   __pyx_k_tuple_14 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_14)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_k_tuple_14);
@@ -25494,20 +26460,34 @@ static int __Pyx_InitCachedConstants(void) {
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__utf8));
   __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_40));
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":79
- *     def __getitem__(self,int k):
- *         if not 0 <= k < self.cs.size():
- *             raise IndexError('candidate set index out of range')             # <<<<<<<<<<<<<<
- *         cdef Candidate candidate = Candidate()
- *         candidate.candidate = &self.cs[0][k]
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":50
+ *     def __getitem__(self, int index):
+ *         if not 0 <= index < len(self):
+ *             raise IndexError('sufficient stats vector index out of range')             # <<<<<<<<<<<<<<
+ *         return self.stats[0][index]
+ * 
  */
-  __pyx_k_tuple_42 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_42)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 79; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_k_tuple_42 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_42)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_k_tuple_42);
   __Pyx_INCREF(((PyObject *)__pyx_kp_s_41));
   PyTuple_SET_ITEM(__pyx_k_tuple_42, 0, ((PyObject *)__pyx_kp_s_41));
   __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_41));
   __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_42));
 
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":84
+ *     def __getitem__(self,int k):
+ *         if not 0 <= k < self.cs.size():
+ *             raise IndexError('candidate set index out of range')             # <<<<<<<<<<<<<<
+ *         cdef Candidate candidate = Candidate()
+ *         candidate.candidate = &self.cs[0][k]
+ */
+  __pyx_k_tuple_44 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_44)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 84; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_44);
+  __Pyx_INCREF(((PyObject *)__pyx_kp_s_43));
+  PyTuple_SET_ITEM(__pyx_k_tuple_44, 0, ((PyObject *)__pyx_kp_s_43));
+  __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_43));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_44));
+
   /* "_cdec.pyx":50
  *         """
  *         if config_str is None:
@@ -25515,15 +26495,15 @@ static int __Pyx_InitCachedConstants(void) {
  *             if formalism not in ('scfg', 'fst', 'lextrans', 'pb',
  *                     'csplit', 'tagger', 'lexalign'):
  */
-  __pyx_k_tuple_45 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_45)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_45);
+  __pyx_k_tuple_47 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_47)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 50; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_47);
   __Pyx_INCREF(((PyObject *)__pyx_n_s__formalism));
-  PyTuple_SET_ITEM(__pyx_k_tuple_45, 0, ((PyObject *)__pyx_n_s__formalism));
+  PyTuple_SET_ITEM(__pyx_k_tuple_47, 0, ((PyObject *)__pyx_n_s__formalism));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__formalism));
   __Pyx_INCREF(Py_None);
-  PyTuple_SET_ITEM(__pyx_k_tuple_45, 1, Py_None);
+  PyTuple_SET_ITEM(__pyx_k_tuple_47, 1, Py_None);
   __Pyx_GIVEREF(Py_None);
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_45));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_47));
 
   /* "_cdec.pyx":88
  *         with open(weights) as fp:
@@ -25532,12 +26512,12 @@ static int __Pyx_InitCachedConstants(void) {
  *                 fname, value = line.split()
  *                 self.weights[fname.strip()] = float(value)
  */
-  __pyx_k_tuple_49 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_49)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_49);
-  __Pyx_INCREF(((PyObject *)__pyx_kp_s_48));
-  PyTuple_SET_ITEM(__pyx_k_tuple_49, 0, ((PyObject *)__pyx_kp_s_48));
-  __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_48));
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_49));
+  __pyx_k_tuple_51 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_51)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 88; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_51);
+  __Pyx_INCREF(((PyObject *)__pyx_kp_s_50));
+  PyTuple_SET_ITEM(__pyx_k_tuple_51, 0, ((PyObject *)__pyx_kp_s_50));
+  __Pyx_GIVEREF(((PyObject *)__pyx_kp_s_50));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_51));
 
   /* "_cdec.pyx":86
  * 
@@ -25546,18 +26526,18 @@ static int __Pyx_InitCachedConstants(void) {
  *             for line in fp:
  *                 if line.strip().startswith('#'): continue
  */
-  __pyx_k_tuple_50 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_50)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_50);
+  __pyx_k_tuple_52 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_52)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 86; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_52);
   __Pyx_INCREF(Py_None);
-  PyTuple_SET_ITEM(__pyx_k_tuple_50, 0, Py_None);
+  PyTuple_SET_ITEM(__pyx_k_tuple_52, 0, Py_None);
   __Pyx_GIVEREF(Py_None);
   __Pyx_INCREF(Py_None);
-  PyTuple_SET_ITEM(__pyx_k_tuple_50, 1, Py_None);
+  PyTuple_SET_ITEM(__pyx_k_tuple_52, 1, Py_None);
   __Pyx_GIVEREF(Py_None);
   __Pyx_INCREF(Py_None);
-  PyTuple_SET_ITEM(__pyx_k_tuple_50, 2, Py_None);
+  PyTuple_SET_ITEM(__pyx_k_tuple_52, 2, Py_None);
   __Pyx_GIVEREF(Py_None);
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_50));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_52));
 
   /* "/Users/vchahun/Sandbox/cdec/python/src/grammar.pxi":3
  * cimport grammar
@@ -25566,44 +26546,44 @@ static int __Pyx_InitCachedConstants(void) {
  *     return ' '.join(w.encode('utf8') if isinstance(w, unicode) else str(w) for w in phrase)
  * 
  */
-  __pyx_k_tuple_52 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_52)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_52);
+  __pyx_k_tuple_54 = PyTuple_New(3); if (unlikely(!__pyx_k_tuple_54)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_54);
   __Pyx_INCREF(((PyObject *)__pyx_n_s__phrase));
-  PyTuple_SET_ITEM(__pyx_k_tuple_52, 0, ((PyObject *)__pyx_n_s__phrase));
+  PyTuple_SET_ITEM(__pyx_k_tuple_54, 0, ((PyObject *)__pyx_n_s__phrase));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__phrase));
   __Pyx_INCREF(((PyObject *)__pyx_n_s__genexpr));
-  PyTuple_SET_ITEM(__pyx_k_tuple_52, 1, ((PyObject *)__pyx_n_s__genexpr));
+  PyTuple_SET_ITEM(__pyx_k_tuple_54, 1, ((PyObject *)__pyx_n_s__genexpr));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__genexpr));
   __Pyx_INCREF(((PyObject *)__pyx_n_s__genexpr));
-  PyTuple_SET_ITEM(__pyx_k_tuple_52, 2, ((PyObject *)__pyx_n_s__genexpr));
+  PyTuple_SET_ITEM(__pyx_k_tuple_54, 2, ((PyObject *)__pyx_n_s__genexpr));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__genexpr));
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_52));
-  __pyx_k_codeobj_53 = (PyObject*)__Pyx_PyCode_New(1, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_52, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_54, __pyx_n_s___phrase, 3, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_53)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_54));
+  __pyx_k_codeobj_55 = (PyObject*)__Pyx_PyCode_New(1, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_54, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_56, __pyx_n_s___phrase, 3, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_55)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":143
- *         return self.name.c_str()
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":190
+ *         return []
  * 
  * BLEU = Scorer('IBM_BLEU')             # <<<<<<<<<<<<<<
  * TER = Scorer('TER')
  */
-  __pyx_k_tuple_55 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_55)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_55);
+  __pyx_k_tuple_57 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_57)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_57);
   __Pyx_INCREF(((PyObject *)__pyx_n_s__IBM_BLEU));
-  PyTuple_SET_ITEM(__pyx_k_tuple_55, 0, ((PyObject *)__pyx_n_s__IBM_BLEU));
+  PyTuple_SET_ITEM(__pyx_k_tuple_57, 0, ((PyObject *)__pyx_n_s__IBM_BLEU));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__IBM_BLEU));
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_55));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_57));
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":144
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":191
  * 
  * BLEU = Scorer('IBM_BLEU')
  * TER = Scorer('TER')             # <<<<<<<<<<<<<<
  */
-  __pyx_k_tuple_56 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_56)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_56);
+  __pyx_k_tuple_58 = PyTuple_New(1); if (unlikely(!__pyx_k_tuple_58)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_58);
   __Pyx_INCREF(((PyObject *)__pyx_n_s__TER));
-  PyTuple_SET_ITEM(__pyx_k_tuple_56, 0, ((PyObject *)__pyx_n_s__TER));
+  PyTuple_SET_ITEM(__pyx_k_tuple_58, 0, ((PyObject *)__pyx_n_s__TER));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__TER));
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_56));
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_58));
 
   /* "_cdec.pyx":28
  * class ParseFailed(Exception): pass
@@ -25612,25 +26592,25 @@ static int __Pyx_InitCachedConstants(void) {
  *     for key, value in config.items():
  *         if isinstance(value, dict):
  */
-  __pyx_k_tuple_57 = PyTuple_New(5); if (unlikely(!__pyx_k_tuple_57)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_k_tuple_57);
+  __pyx_k_tuple_59 = PyTuple_New(5); if (unlikely(!__pyx_k_tuple_59)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_k_tuple_59);
   __Pyx_INCREF(((PyObject *)__pyx_n_s__config));
-  PyTuple_SET_ITEM(__pyx_k_tuple_57, 0, ((PyObject *)__pyx_n_s__config));
+  PyTuple_SET_ITEM(__pyx_k_tuple_59, 0, ((PyObject *)__pyx_n_s__config));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__config));
   __Pyx_INCREF(((PyObject *)__pyx_n_s__key));
-  PyTuple_SET_ITEM(__pyx_k_tuple_57, 1, ((PyObject *)__pyx_n_s__key));
+  PyTuple_SET_ITEM(__pyx_k_tuple_59, 1, ((PyObject *)__pyx_n_s__key));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__key));
   __Pyx_INCREF(((PyObject *)__pyx_n_s__value));
-  PyTuple_SET_ITEM(__pyx_k_tuple_57, 2, ((PyObject *)__pyx_n_s__value));
+  PyTuple_SET_ITEM(__pyx_k_tuple_59, 2, ((PyObject *)__pyx_n_s__value));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__value));
   __Pyx_INCREF(((PyObject *)__pyx_n_s__name));
-  PyTuple_SET_ITEM(__pyx_k_tuple_57, 3, ((PyObject *)__pyx_n_s__name));
+  PyTuple_SET_ITEM(__pyx_k_tuple_59, 3, ((PyObject *)__pyx_n_s__name));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__name));
   __Pyx_INCREF(((PyObject *)__pyx_n_s__info));
-  PyTuple_SET_ITEM(__pyx_k_tuple_57, 4, ((PyObject *)__pyx_n_s__info));
+  PyTuple_SET_ITEM(__pyx_k_tuple_59, 4, ((PyObject *)__pyx_n_s__info));
   __Pyx_GIVEREF(((PyObject *)__pyx_n_s__info));
-  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_57));
-  __pyx_k_codeobj_58 = (PyObject*)__Pyx_PyCode_New(1, 0, 5, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_57, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_59, __pyx_n_s___make_config, 28, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_58)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_59));
+  __pyx_k_codeobj_60 = (PyObject*)__Pyx_PyCode_New(1, 0, 5, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_k_tuple_59, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_61, __pyx_n_s___make_config, 28, __pyx_empty_bytes); if (unlikely(!__pyx_k_codeobj_60)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_RefNannyFinishContext();
   return 0;
   __pyx_L1_error:;
@@ -25764,15 +26744,18 @@ PyMODINIT_FUNC PyInit__cdec(void)
   if (PyType_Ready(&__pyx_type_5_cdec_SufficientStats) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   if (__Pyx_SetAttrString(__pyx_m, "SufficientStats", (PyObject *)&__pyx_type_5_cdec_SufficientStats) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec_SufficientStats = &__pyx_type_5_cdec_SufficientStats;
-  if (PyType_Ready(&__pyx_type_5_cdec_CandidateSet) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  if (__Pyx_SetAttrString(__pyx_m, "CandidateSet", (PyObject *)&__pyx_type_5_cdec_CandidateSet) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyType_Ready(&__pyx_type_5_cdec_CandidateSet) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__Pyx_SetAttrString(__pyx_m, "CandidateSet", (PyObject *)&__pyx_type_5_cdec_CandidateSet) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec_CandidateSet = &__pyx_type_5_cdec_CandidateSet;
-  if (PyType_Ready(&__pyx_type_5_cdec_SegmentEvaluator) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  if (__Pyx_SetAttrString(__pyx_m, "SegmentEvaluator", (PyObject *)&__pyx_type_5_cdec_SegmentEvaluator) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 93; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyType_Ready(&__pyx_type_5_cdec_SegmentEvaluator) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__Pyx_SetAttrString(__pyx_m, "SegmentEvaluator", (PyObject *)&__pyx_type_5_cdec_SegmentEvaluator) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 98; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec_SegmentEvaluator = &__pyx_type_5_cdec_SegmentEvaluator;
-  if (PyType_Ready(&__pyx_type_5_cdec_Scorer) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  if (__Pyx_SetAttrString(__pyx_m, "Scorer", (PyObject *)&__pyx_type_5_cdec_Scorer) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyType_Ready(&__pyx_type_5_cdec_Scorer) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__Pyx_SetAttrString(__pyx_m, "Scorer", (PyObject *)&__pyx_type_5_cdec_Scorer) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 117; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec_Scorer = &__pyx_type_5_cdec_Scorer;
+  if (PyType_Ready(&__pyx_type_5_cdec_Metric) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (__Pyx_SetAttrString(__pyx_m, "Metric", (PyObject *)&__pyx_type_5_cdec_Metric) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 173; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_ptype_5_cdec_Metric = &__pyx_type_5_cdec_Metric;
   if (PyType_Ready(&__pyx_type_5_cdec_Decoder) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   if (__Pyx_SetAttrString(__pyx_m, "Decoder", (PyObject *)&__pyx_type_5_cdec_Decoder) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec_Decoder = &__pyx_type_5_cdec_Decoder;
@@ -25818,7 +26801,7 @@ PyMODINIT_FUNC PyInit__cdec(void)
   __pyx_ptype_5_cdec___pyx_scope_struct_19_lines = &__pyx_type_5_cdec___pyx_scope_struct_19_lines;
   if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_20___iter__) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec___pyx_scope_struct_20___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_20___iter__;
-  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_21___iter__) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 85; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_21___iter__) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec___pyx_scope_struct_21___iter__ = &__pyx_type_5_cdec___pyx_scope_struct_21___iter__;
   if (PyType_Ready(&__pyx_type_5_cdec___pyx_scope_struct_22__make_config) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_5_cdec___pyx_scope_struct_22__make_config = &__pyx_type_5_cdec___pyx_scope_struct_22__make_config;
@@ -25843,25 +26826,25 @@ PyMODINIT_FUNC PyInit__cdec(void)
   if (PyObject_SetAttr(__pyx_m, __pyx_n_s___phrase, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":143
- *         return self.name.c_str()
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":190
+ *         return []
  * 
  * BLEU = Scorer('IBM_BLEU')             # <<<<<<<<<<<<<<
  * TER = Scorer('TER')
  */
-  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Scorer)), ((PyObject *)__pyx_k_tuple_55), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Scorer)), ((PyObject *)__pyx_k_tuple_57), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  if (PyObject_SetAttr(__pyx_m, __pyx_n_s__BLEU, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyObject_SetAttr(__pyx_m, __pyx_n_s__BLEU, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
 
-  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":144
+  /* "/Users/vchahun/Sandbox/cdec/python/src/mteval.pxi":191
  * 
  * BLEU = Scorer('IBM_BLEU')
  * TER = Scorer('TER')             # <<<<<<<<<<<<<<
  */
-  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Scorer)), ((PyObject *)__pyx_k_tuple_56), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_5_cdec_Scorer)), ((PyObject *)__pyx_k_tuple_58), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  if (PyObject_SetAttr(__pyx_m, __pyx_n_s__TER, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyObject_SetAttr(__pyx_m, __pyx_n_s__TER, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
 
   /* "_cdec.pyx":22
@@ -26356,6 +27339,40 @@ static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject*
     }
 }
 
+static CYTHON_INLINE int __Pyx_CheckKeywordStrings(
+    PyObject *kwdict,
+    const char* function_name,
+    int kw_allowed)
+{
+    PyObject* key = 0;
+    Py_ssize_t pos = 0;
+    while (PyDict_Next(kwdict, &pos, &key, 0)) {
+        #if PY_MAJOR_VERSION < 3
+        if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key)))
+        #else
+        if (unlikely(!PyUnicode_Check(key)))
+        #endif
+            goto invalid_keyword_type;
+    }
+    if ((!kw_allowed) && unlikely(key))
+        goto invalid_keyword;
+    return 1;
+invalid_keyword_type:
+    PyErr_Format(PyExc_TypeError,
+        "%s() keywords must be strings", function_name);
+    return 0;
+invalid_keyword:
+    PyErr_Format(PyExc_TypeError,
+    #if PY_MAJOR_VERSION < 3
+        "%s() got an unexpected keyword argument '%s'",
+        function_name, PyString_AsString(key));
+    #else
+        "%s() got an unexpected keyword argument '%U'",
+        function_name, key);
+    #endif
+    return 0;
+}
+
 static double __Pyx__PyObject_AsDouble(PyObject* obj) {
     PyObject* float_value;
     if (Py_TYPE(obj)->tp_as_number && Py_TYPE(obj)->tp_as_number->nb_float) {
diff --git a/python/src/decoder.pxd b/python/src/decoder.pxd
index d2065579..a66166a2 100644
--- a/python/src/decoder.pxd
+++ b/python/src/decoder.pxd
@@ -27,8 +27,6 @@ cdef extern from "decoder/decoder.h":
         variables_map& GetConf()
 
         # add grammar rules (currently only supported by SCFG decoders)
-        # that will be used on subsequent calls to Decode. rules should be in standard
-        # text format. This function does NOT read from a file.
         void AddSupplementalGrammarFromString(string& grammar_str)
         void AddSupplementalGrammar(shared_ptr[Grammar] grammar)
 
diff --git a/python/src/grammar.pxd b/python/src/grammar.pxd
index 43806f71..8853a614 100644
--- a/python/src/grammar.pxd
+++ b/python/src/grammar.pxd
@@ -3,7 +3,6 @@ from libcpp.string cimport string
 from utils cimport *
 
 cdef extern from "decoder/trule.h":
-
     cdef cppclass AlignmentPoint:
         AlignmentPoint(int s, int t)
         AlignmentPoint Inverted()
diff --git a/python/src/mteval.pxd b/python/src/mteval.pxd
index 52af6297..27c2808d 100644
--- a/python/src/mteval.pxd
+++ b/python/src/mteval.pxd
@@ -10,6 +10,7 @@ cdef extern from "mteval/ns.h":
         unsigned size()
         float operator[](unsigned i)
         void swap(SufficientStats& other)
+        vector[float] fields
 
     SufficientStats add "operator+" (SufficientStats&, SufficientStats&)
 
@@ -26,9 +27,15 @@ cdef extern from "mteval/ns.h":
                                     vector[WordID]& refs,
                                     SufficientStats* out)
 
-cdef extern from "mteval/ns.h" namespace "EvaluationMetric":
-    EvaluationMetric* Instance(string& metric_id)
-    EvaluationMetric* Instance() # IBM_BLEU
+    cdef EvaluationMetric* MetricInstance "EvaluationMetric::Instance" (string& metric_id)
+
+cdef extern from "py_scorer.h":
+    ctypedef float (*MetricScoreCallback)(void*, SufficientStats* stats)
+    ctypedef void (*MetricStatsCallback)(void*, 
+            string* hyp, vector[string]* refs, SufficientStats* out)
+    
+    cdef EvaluationMetric* PyMetricInstance "PythonEvaluationMetric::Instance"(
+            string& metric_id, void*, MetricStatsCallback, MetricScoreCallback)
 
 cdef extern from "training/candidate_set.h" namespace "training":
     cdef cppclass Candidate "const training::Candidate":
diff --git a/python/src/mteval.pxi b/python/src/mteval.pxi
index 4ba73168..f67f4f04 100644
--- a/python/src/mteval.pxi
+++ b/python/src/mteval.pxi
@@ -43,7 +43,12 @@ cdef class SufficientStats:
 
     def __iter__(self):
         for i in range(len(self)):
-            yield self.stats[0][i]
+            yield self[i]
+
+    def __getitem__(self, int index):
+        if not 0 <= index < len(self):
+            raise IndexError('sufficient stats vector index out of range')
+        return self.stats[0][index]
 
     def __iadd__(SufficientStats self, SufficientStats other):
         self.stats[0] += other.stats[0]
@@ -111,15 +116,17 @@ cdef class SegmentEvaluator:
 
 cdef class Scorer:
     cdef string* name
+    cdef mteval.EvaluationMetric* metric
 
-    def __cinit__(self, char* name):
-        self.name = new string(name)
+    def __cinit__(self, bytes name=None):
+        if name:
+            self.name = new string(name)
+            self.metric = mteval.MetricInstance(self.name[0])
 
     def __dealloc__(self):
         del self.name
     
     def __call__(self, refs):
-        cdef mteval.EvaluationMetric* metric = mteval.Instance(self.name[0])
         if isinstance(refs, unicode) or isinstance(refs, str):
             refs = [refs]
         cdef vector[vector[WordID]]* refsv = new vector[vector[WordID]]()
@@ -132,13 +139,53 @@ cdef class Scorer:
             del refv
         cdef unsigned i
         cdef SegmentEvaluator evaluator = SegmentEvaluator()
-        evaluator.metric = metric
-        evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](metric.CreateSegmentEvaluator(refsv[0]))
+        evaluator.metric = self.metric
+        evaluator.scorer = new shared_ptr[mteval.SegmentEvaluator](
+                self.metric.CreateSegmentEvaluator(refsv[0]))
         del refsv # in theory should not delete but store in SegmentEvaluator
         return evaluator
 
     def __str__(self):
         return self.name.c_str()
 
+cdef float _compute_score(void* metric_, mteval.SufficientStats* stats):
+    cdef Metric metric = <Metric> metric_
+    cdef list ss = []
+    cdef unsigned i
+    for i in range(stats.size()):
+        ss.append(stats[0][i])
+    return metric.score(ss)
+
+cdef void _compute_sufficient_stats(void* metric_, 
+        string* hyp,
+        vector[string]* refs,
+        mteval.SufficientStats* out):
+    cdef Metric metric = <Metric> metric_
+    cdef list refs_ = []
+    cdef unsigned i
+    for i in range(refs.size()):
+        refs_.append(refs[0][i].c_str())
+    cdef list ss = metric.evaluate(hyp.c_str(), refs_)
+    out.fields.resize(len(ss))
+    for i in range(len(ss)):
+        out.fields[i] = ss[i]
+
+cdef class Metric:
+    cdef Scorer scorer
+    def __cinit__(self):
+        self.scorer = Scorer()
+        self.scorer.name = new string(as_str(self.__class__.__name__))
+        self.scorer.metric = mteval.PyMetricInstance(self.scorer.name[0],
+                <void*> self, _compute_sufficient_stats, _compute_score)
+
+    def __call__(self, refs):
+        return self.scorer(refs)
+
+    def score(SufficientStats stats):
+        return 0
+
+    def evaluate(self, hyp, refs):
+        return []
+
 BLEU = Scorer('IBM_BLEU')
 TER = Scorer('TER')
diff --git a/python/src/observer.h b/python/src/observer.h
index d398f2f7..05f3c9be 100644
--- a/python/src/observer.h
+++ b/python/src/observer.h
@@ -1,6 +1,5 @@
 #include "decoder/hg.h"
 #include "decoder/decoder.h"
-#include <iostream>
 
 struct BasicObserver: public DecoderObserver {
     Hypergraph* hypergraph;
diff --git a/python/src/py_scorer.h b/python/src/py_scorer.h
new file mode 100644
index 00000000..22dc9fee
--- /dev/null
+++ b/python/src/py_scorer.h
@@ -0,0 +1,44 @@
+#include "ns.h"
+#include "tdict.h"
+
+typedef float (*MetricScoreCallback)(void*, SufficientStats* stats);
+typedef void (*MetricStatsCallback)(void*,
+        std::string *hyp,
+        std::vector<std::string> *refs,
+        SufficientStats* out);
+
+struct PythonEvaluationMetric : public EvaluationMetric {
+
+    PythonEvaluationMetric(const std::string& id) : EvaluationMetric(id) {}
+
+    static EvaluationMetric* Instance(const std::string& id, 
+            void* obj,
+            MetricStatsCallback statscb,
+            MetricScoreCallback scorecb) {
+        PythonEvaluationMetric* metric = new PythonEvaluationMetric(id);
+        metric->pymetric = obj;
+        metric->_compute_score =  scorecb;
+        metric->_compute_sufficient_stats = statscb;
+        return metric;
+    }
+
+    float ComputeScore(const SufficientStats& stats) const {
+        SufficientStats stats_(stats);
+        return _compute_score(pymetric, &stats_);
+    }
+
+    void ComputeSufficientStatistics(const std::vector<WordID>& hyp,
+            const std::vector<std::vector<WordID> >& refs,
+            SufficientStats* out) const {
+        std::string hyp_(TD::GetString(hyp));
+        std::vector<std::string> refs_;
+        for(unsigned i = 0; i < refs.size(); ++i) {
+            refs_.push_back(TD::GetString(refs[i]));
+        }
+        _compute_sufficient_stats(pymetric, &hyp_, &refs_, out);
+    }
+
+    void* pymetric;
+    MetricStatsCallback _compute_sufficient_stats;
+    MetricScoreCallback _compute_score;
+};