Merge branch 'net' into origin/net

26 files changed, 2063 insertions, 1017 deletions

diff --git a/.gitignore b/.gitignore
index dde591c7..00d712e4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,7 +53,7 @@ decoder/t2s_test
 decoder/trule_test
 decoder/weights_test
 decoder/network_decoder
-decoder/local
+decoder/feed
 depcomp
 dist
 dpmert/Makefile
@@ -189,6 +189,7 @@ training/test_ngram
 training/const_reorder/argument_reorder_model_trainer
 training/const_reorder/const_reorder_model_trainer
 utils/atools
+utils/atools_net
 utils/bin/
 utils/crp_test
 utils/dict_test
@@ -217,6 +218,7 @@ training/dpmert/sentclient
 training/dpmert/sentserver
 training/dtrain/dtrain
 training/dtrain/dtrain_net
+training/dtrain/dtrain_net_interface
 training/latent_svm/latent_svm
 training/minrisk/minrisk_optimize
 training/mira/ada_opt_sm
@@ -232,4 +234,5 @@ training/utils/sentserver
 utils/stringlib_test
 word-aligner/binderiv
 word-aligner/fast_align
+word-aligner/net_fa
 test-driver
diff --git a/decoder/Makefile.am b/decoder/Makefile.am
index 01d4be14..4c05b8f4 100644
--- a/decoder/Makefile.am
+++ b/decoder/Makefile.am
@@ -1,4 +1,4 @@
-bin_PROGRAMS = cdec minimal_decoder network_decoder local
+bin_PROGRAMS = cdec minimal_decoder network_decoder
 
 noinst_PROGRAMS = \
   trule_test \
diff --git a/decoder/README.net b/decoder/README.net
new file mode 100644
index 00000000..05bb66d8
--- /dev/null
+++ b/decoder/README.net
@@ -0,0 +1,8 @@
+to run the server do
+cd ../training/dtrain/examples/standard/
+../../../../decoder/network_decoder cdec.ini 
+
+to run the 'feeder' do
+cd ../training/dtrain/examples/standard/
+zcat nc-wmt11.100.gz | ../../../../decoder/feed.rb
+
diff --git a/decoder/local.cc b/decoder/local.cc
deleted file mode 100644
index b03fd17e..00000000
--- a/decoder/local.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-#include <iostream>
-#include <sstream>
-#include <stdio.h>
-#include <unistd.h>
-
-#include <nanomsg/nn.h>
-#include <nanomsg/pair.h>
-#include "nn.hpp"
-
-using namespace std;
-
-void
-recv(nn::socket& sock)
-{
-  char *buf = NULL;
-  sock.recv(&buf, NN_MSG, 0);
-  if (buf) {
-    string translation(buf);
-    cout << "received translation '" << translation << "'" << endl;
-  }
-}
-
-void
-send(nn::socket& sock, const string& msg)
-{
-  cout << "sending source '" << msg << "'" << endl;
-  sock.send(msg.c_str(), msg.size()+1, 0);
-}
-
-void
-loop(nn::socket& sock)
-{
-  int to = 100;
-  sock.setsockopt(NN_SOL_SOCKET, NN_RCVTIMEO, &to, sizeof(to));
-  while(1) {
-    send(sock, "das ist ein test .");
-    sleep(1);
-    recv(sock);
-  }
-}
-
-int main(int argc, char const* argv[])
-{
-  nn::socket sock(AF_SP, NN_PAIR);
-  string url = "ipc:///tmp/network_decoder.ipc";
-  sock.connect(url.c_str());
-  loop(sock);
-
-  return 0;
-}
-
diff --git a/decoder/network_decoder.cc b/decoder/network_decoder.cc
index aaa1842d..ebbb91b5 100644
--- a/decoder/network_decoder.cc
+++ b/decoder/network_decoder.cc
@@ -28,7 +28,7 @@ struct TheObserver : public DecoderObserver
 
 int send(nn::socket& sock, const string trans)
 {
-  cout << "sending translation '" << trans << "'" << endl;
+  cout << "sending translation '" << trans << "'" << endl << endl;
   sock.send(trans.c_str(), trans.size()+1, 0);
 }
 
@@ -36,10 +36,11 @@ bool
 recv(nn::socket& sock, string& source)
 {
   char *buf = NULL;
-  sock.recv(&buf, NN_MSG, 0);
+  size_t sz = sock.recv(&buf, NN_MSG, 0);
   if (buf) {
-    string s(buf);
+    string s(buf, buf+sz);
     source = s;
+    nn::freemsg(buf);
 
     return true;
   }
@@ -62,6 +63,8 @@ loop(Decoder& decoder, nn::socket& sock)
       cout << "received source '" << source << "'" << endl;
       decoder.Decode(source, &o);
       send(sock, o.translation);
+    } else {
+      // Oh no!
     }
   }
 }
@@ -75,7 +78,8 @@ main(int argc, char** argv)
   SetSilent(true);
 
   nn::socket sock(AF_SP, NN_PAIR);
-  string url = "ipc:///tmp/network_decoder.ipc";
+  //string url = "ipc:///tmp/network_decoder.ipc";
+  string url = "tcp://127.0.0.1:60666";
   sock.bind(url.c_str());
 
   loop(decoder, sock);
diff --git a/decoder/nn.hpp b/decoder/nn.hpp
deleted file mode 100644
index 50b8304c..00000000
--- a/decoder/nn.hpp
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
-    Copyright (c) 2013 250bpm s.r.o.
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"),
-    to deal in the Software without restriction, including without limitation
-    the rights to use, copy, modify, merge, publish, distribute, sublicense,
-    and/or sell copies of the Software, and to permit persons to whom
-    the Software is furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be included
-    in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-    IN THE SOFTWARE.
-*/
-
-#ifndef NN_HPP_INCLUDED
-#define NN_HPP_INCLUDED
-
-#include <nanomsg/nn.h>
-
-#include <cassert>
-#include <cstring>
-#include <algorithm>
-#include <exception>
-
-#if defined __GNUC__
-#define nn_slow(x) __builtin_expect ((x), 0)
-#else
-#define nn_slow(x) (x)
-#endif
-
-namespace nn
-{
-
-    class exception : public std::exception
-    {
-    public:
-
-        exception () : err (nn_errno ()) {}
-
-        virtual const char *what () const throw ()
-        {
-            return nn_strerror (err);
-        }
-
-        int num () const
-        {
-            return err;
-        }
-
-    private:
-
-        int err;
-    };
-
-    inline const char *symbol (int i, int *value)
-    {
-        return nn_symbol (i, value);
-    }
-
-    inline void *allocmsg (size_t size, int type)
-    {
-        void *msg = nn_allocmsg (size, type);
-        if (nn_slow (!msg))
-            throw nn::exception ();
-        return msg;
-    }
-
-    inline int freemsg (void *msg)
-    {
-        int rc = nn_freemsg (msg);
-        if (nn_slow (rc != 0))
-            throw nn::exception ();
-        return rc;
-    }
-
-    class socket
-    {
-    public:
-
-        inline socket (int domain, int protocol)
-        {
-            s = nn_socket (domain, protocol);
-            if (nn_slow (s < 0))
-                throw nn::exception ();
-        }
-
-        inline ~socket ()
-        {
-            int rc = nn_close (s);
-            assert (rc == 0);
-        }
-
-        inline void setsockopt (int level, int option, const void *optval,
-            size_t optvallen)
-        {
-            int rc = nn_setsockopt (s, level, option, optval, optvallen);
-            if (nn_slow (rc != 0))
-                throw nn::exception ();
-        }
-
-        inline void getsockopt (int level, int option, void *optval,
-            size_t *optvallen)
-        {
-            int rc = nn_getsockopt (s, level, option, optval, optvallen);
-            if (nn_slow (rc != 0))
-                throw nn::exception ();
-        }
-
-        inline int bind (const char *addr)
-        {
-            int rc = nn_bind (s, addr);
-            if (nn_slow (rc < 0))
-                throw nn::exception ();
-            return rc;
-        }
-
-        inline int connect (const char *addr)
-        {
-            int rc = nn_connect (s, addr);
-            if (nn_slow (rc < 0))
-                throw nn::exception ();
-            return rc;
-        }
-
-        inline void shutdown (int how)
-        {
-            int rc = nn_shutdown (s, how);
-            if (nn_slow (rc != 0))
-                throw nn::exception ();
-        }
-
-        inline int send (const void *buf, size_t len, int flags)
-        {
-            int rc = nn_send (s, buf, len, flags);
-            if (nn_slow (rc < 0)) {
-                if (nn_slow (nn_errno () != EAGAIN))
-                    throw nn::exception ();
-                return -1;
-            }
-            return rc;
-        }
-
-        inline int recv (void *buf, size_t len, int flags)
-        {
-            int rc = nn_recv (s, buf, len, flags);
-            if (nn_slow (rc < 0)) {
-                if (nn_slow (nn_errno () != EAGAIN))
-                    throw nn::exception ();
-                return -1;
-            }
-            return rc;
-        }
-
-        inline int sendmsg (const struct nn_msghdr *msghdr, int flags)
-        {
-            int rc = nn_sendmsg (s, msghdr, flags);
-            if (nn_slow (rc < 0)) {
-                if (nn_slow (nn_errno () != EAGAIN))
-                    throw nn::exception ();
-                return -1;
-            }
-            return rc;
-        }
-
-        inline int recvmsg (struct nn_msghdr *msghdr, int flags)
-        {
-            int rc = nn_recvmsg (s, msghdr, flags);
-            if (nn_slow (rc < 0)) {
-                if (nn_slow (nn_errno () != EAGAIN))
-                    throw nn::exception ();
-                return -1;
-            }
-            return rc;
-        }
-
-    private:
-
-        int s;
-
-        /*  Prevent making copies of the socket by accident. */
-        socket (const socket&);
-        void operator = (const socket&);
-    };
-
-    inline void term ()
-    {
-        nn_term ();
-    }
-
-}
-
-#undef nn_slow
-
-#endif
-
-
diff --git a/feed.rb b/feed.rb
new file mode 100755
index 00000000..a925d31c
--- /dev/null
+++ b/feed.rb
@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'nanomsg'
+require 'trollop'
+
+conf = Trollop::options do
+  opt :addr, "URL of socket", :type => :string, :short => '-a', :default => "tcp://127.0.0.1:31337"
+end
+
+sock = NanoMsg::PairSocket.new
+addr = conf[:addr]
+sock.connect addr
+puts "< got #{sock.recv}"
+
+while true
+  line = STDIN.gets
+  if !line || line.strip=='shutdown'
+    puts "shutting down"
+    sock.send 'shutdown'
+    break
+  end
+  puts "sending '#{line.strip}'"
+  sock.send line.strip
+  sleep 1
+  puts "got response '#{sock.recv}'"
+end
+
diff --git a/python/cdec/_cdec.cpp b/python/cdec/_cdec.cpp
index e76eb468..872ef5b2 100644
--- a/python/cdec/_cdec.cpp
+++ b/python/cdec/_cdec.cpp
@@ -1,4 +1,90 @@
-/* Generated by Cython 0.21 */
+/* Generated by Cython 0.22 */
+
+/* BEGIN: Cython Metadata
+{
+    "distutils": {
+        "language": "c++", 
+        "libraries": [
+            "cdec", 
+            "utils", 
+            "mteval", 
+            "training_utils", 
+            "klm", 
+            "klm_util", 
+            "klm_util_double", 
+            "ksearch", 
+            "dl", 
+            "rt", 
+            "boost_program_options-mt", 
+            "boost_regex-mt", 
+            "boost_serialization-mt", 
+            "boost_system-mt", 
+            "boost_filesystem-mt", 
+            "z", 
+            "bz2", 
+            "lzma"
+        ], 
+        "depends": [
+            "../utils/verbose.h", 
+            "../utils/sampler.h", 
+            "../utils/tdict.h", 
+            "../decoder/ff_register.h", 
+            "../decoder/inside_outside.h", 
+            "../decoder/hg_io.h", 
+            "../utils/filelib.h", 
+            "../decoder/kbest.h", 
+            "../decoder/csplit.h", 
+            "../utils/logval.h", 
+            "../decoder/grammar.h", 
+            "../utils/weights.h", 
+            "../decoder/trule.h", 
+            "../decoder/viterbi.h", 
+            "../decoder/hg_intersect.h", 
+            "../decoder/decoder.h", 
+            "cdec/observer.h", 
+            "../utils/small_vector.h", 
+            "cdec/py_scorer.h", 
+            "../utils/fdict.h", 
+            "../mteval/ns.h", 
+            "../utils/wordid.h", 
+            "../decoder/hg_sampler.h", 
+            "../decoder/lattice.h", 
+            "../decoder/hg.h", 
+            "../training/utils/candidate_set.h", 
+            "../utils/sparse_vector.h"
+        ], 
+        "extra_compile_args": [
+            "-DPIC", 
+            "-pthread", 
+            "-DHAVE_CONFIG_H", 
+            "-DKENLM_MAX_ORDER=6", 
+            "-std=gnu++11", 
+            "-march=native", 
+            "-mtune=native", 
+            "-fPIC", 
+            "-g", 
+            "-O3"
+        ], 
+        "library_dirs": [
+            "../decoder", 
+            "../utils", 
+            "../mteval", 
+            "../training/utils", 
+            "../klm/lm", 
+            "../klm/util", 
+            "../klm/util/double-conversion", 
+            "../klm/search"
+        ], 
+        "include_dirs": [
+            "..", 
+            "cdec/", 
+            "../decoder", 
+            "../utils", 
+            "../mteval"
+        ]
+    }
+}
+END: Cython Metadata */
 
 #define PY_SSIZE_T_CLEAN
 #ifndef CYTHON_USE_PYLONG_INTERNALS
@@ -19,7 +105,7 @@
 #elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000)
     #error Cython requires Python 2.6+ or Python 3.2+.
 #else
-#define CYTHON_ABI "0_21"
+#define CYTHON_ABI "0_22"
 #include <stddef.h>
 #ifndef offsetof
 #define offsetof(type, member) ( (size_t) & ((type*)0) -> member )
@@ -54,7 +140,7 @@
 #define CYTHON_COMPILING_IN_PYPY 0
 #define CYTHON_COMPILING_IN_CPYTHON 1
 #endif
-#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600
+#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag)
 #define Py_OptimizeFlag 0
 #endif
 #define __PYX_BUILD_PY_SSIZE_T "n"
@@ -73,8 +159,6 @@
 #if PY_MAJOR_VERSION >= 3
   #define Py_TPFLAGS_CHECKTYPES 0
   #define Py_TPFLAGS_HAVE_INDEX 0
-#endif
-#if PY_MAJOR_VERSION >= 3
   #define Py_TPFLAGS_HAVE_NEWBUFFER 0
 #endif
 #if PY_VERSION_HEX < 0x030400a1 && !defined(Py_TPFLAGS_HAVE_FINALIZE)
@@ -101,10 +185,12 @@
 #if CYTHON_COMPILING_IN_PYPY
   #define __Pyx_PyUnicode_Concat(a, b)      PyNumber_Add(a, b)
   #define __Pyx_PyUnicode_ConcatSafe(a, b)  PyNumber_Add(a, b)
+  #define __Pyx_PyFrozenSet_Size(s)         PyObject_Size(s)
 #else
   #define __Pyx_PyUnicode_Concat(a, b)      PyUnicode_Concat(a, b)
   #define __Pyx_PyUnicode_ConcatSafe(a, b)  ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \
       PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b))
+  #define __Pyx_PyFrozenSet_Size(s)         PySet_Size(s)
 #endif
 #define __Pyx_PyString_FormatSafe(a, b)   ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b))
 #define __Pyx_PyUnicode_FormatSafe(a, b)  ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b))
@@ -151,6 +237,11 @@
 #if PY_MAJOR_VERSION >= 3
   #define PyBoolObject                 PyLongObject
 #endif
+#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY
+  #ifndef PyUnicode_InternFromString
+    #define PyUnicode_InternFromString(s) PyUnicode_FromString(s)
+  #endif
+#endif
 #if PY_VERSION_HEX < 0x030200A4
   typedef long Py_hash_t;
   #define __Pyx_PyInt_FromHash_t PyInt_FromLong
@@ -160,7 +251,9 @@
   #define __Pyx_PyInt_AsHash_t   PyInt_AsSsize_t
 #endif
 #if PY_MAJOR_VERSION >= 3
-  #define PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func))
+  #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func))
+#else
+  #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass)
 #endif
 #ifndef CYTHON_INLINE
   #if defined(__GNUC__)
@@ -196,11 +289,22 @@ static CYTHON_INLINE float __PYX_NAN() {
   return value;
 }
 #endif
+#define __Pyx_void_to_None(void_result) (void_result, Py_INCREF(Py_None), Py_None)
 #ifdef __cplusplus
 template<typename T>
 void __Pyx_call_destructor(T* x) {
     x->~T();
 }
+template<typename T>
+class __Pyx_FakeReference {
+  public:
+    __Pyx_FakeReference() : ptr(NULL) { }
+    __Pyx_FakeReference(T& ref) : ptr(&ref) { }
+    T *operator->() { return ptr; }
+    operator T&() { return *ptr; }
+  private:
+    T *ptr;
+};
 #endif
 
 
@@ -320,11 +424,11 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*);
 #endif
 #define __Pyx_PyObject_AsSString(s)    ((signed char*) __Pyx_PyObject_AsString(s))
 #define __Pyx_PyObject_AsUString(s)    ((unsigned char*) __Pyx_PyObject_AsString(s))
-#define __Pyx_PyObject_FromUString(s)  __Pyx_PyObject_FromString((const char*)s)
-#define __Pyx_PyBytes_FromUString(s)   __Pyx_PyBytes_FromString((const char*)s)
-#define __Pyx_PyByteArray_FromUString(s)   __Pyx_PyByteArray_FromString((const char*)s)
-#define __Pyx_PyStr_FromUString(s)     __Pyx_PyStr_FromString((const char*)s)
-#define __Pyx_PyUnicode_FromUString(s) __Pyx_PyUnicode_FromString((const char*)s)
+#define __Pyx_PyObject_FromCString(s)  __Pyx_PyObject_FromString((const char*)s)
+#define __Pyx_PyBytes_FromCString(s)   __Pyx_PyBytes_FromString((const char*)s)
+#define __Pyx_PyByteArray_FromCString(s)   __Pyx_PyByteArray_FromString((const char*)s)
+#define __Pyx_PyStr_FromCString(s)     __Pyx_PyStr_FromString((const char*)s)
+#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s)
 #if PY_MAJOR_VERSION < 3
 static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u)
 {
@@ -360,7 +464,7 @@ static int __Pyx_init_sys_getdefaultencoding_params(void) {
     const char* default_encoding_c;
     sys = PyImport_ImportModule("sys");
     if (!sys) goto bad;
-    default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL);
+    default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL);
     Py_DECREF(sys);
     if (!default_encoding) goto bad;
     default_encoding_c = PyBytes_AsString(default_encoding);
@@ -759,8 +863,8 @@ struct __pyx_obj_4cdec_5_cdec_HypergraphNode {
 };
 
 
-/* "cdec/lattice.pxi":3
- * cimport lattice
+/* "cdec/lattice.pxi":4
+ * from utils cimport *
  * 
  * cdef class Lattice:             # <<<<<<<<<<<<<<
  *     cdef lattice.Lattice* lattice
@@ -1258,7 +1362,7 @@ struct __pyx_obj_4cdec_5_cdec___pyx_scope_struct_21___get__ {
 };
 
 
-/* "cdec/lattice.pxi":56
+/* "cdec/lattice.pxi":58
  *         return unicode(str(self), 'utf8')
  * 
  *     def __iter__(self):             # <<<<<<<<<<<<<<
@@ -1274,7 +1378,7 @@ struct __pyx_obj_4cdec_5_cdec___pyx_scope_struct_22___iter__ {
 };
 
 
-/* "cdec/lattice.pxi":61
+/* "cdec/lattice.pxi":63
  *             yield self[i]
  * 
  *     def todot(self):             # <<<<<<<<<<<<<<
@@ -1287,7 +1391,7 @@ struct __pyx_obj_4cdec_5_cdec___pyx_scope_struct_23_todot {
 };
 
 
-/* "cdec/lattice.pxi":63
+/* "cdec/lattice.pxi":65
  *     def todot(self):
  *         """lattice.todot() -> Representation of the lattice in GraphViz dot format."""
  *         def lines():             # <<<<<<<<<<<<<<
@@ -1487,6 +1591,8 @@ struct __pyx_vtabstruct_4cdec_5_cdec_HypergraphNode {
   PyObject *(*init)(struct __pyx_obj_4cdec_5_cdec_HypergraphNode *, Hypergraph *, unsigned int);
 };
 static struct __pyx_vtabstruct_4cdec_5_cdec_HypergraphNode *__pyx_vtabptr_4cdec_5_cdec_HypergraphNode;
+
+/* --- Runtime support code (head) --- */
 #ifndef CYTHON_REFNANNY
   #define CYTHON_REFNANNY 0
 #endif
@@ -1845,14 +1951,12 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object);
 static void __Pyx_AddTraceback(const char *funcname, int c_line,
                                int py_line, const char *filename);
 
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value);
-
-static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *);
-
 static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *);
 
 static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level);
 
+static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *);
+
 #ifndef __Pyx_CppExn2PyErr
 #include <new>
 #include <typeinfo>
@@ -1892,6 +1996,8 @@ static void __Pyx_CppExn2PyErr() {
 }
 #endif
 
+static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value);
+
 static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value);
 
 static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value);
@@ -2049,7 +2155,7 @@ static struct __pyx_obj_4cdec_5_cdec_TRule *__pyx_f_4cdec_5_cdec_convert_rule(st
 static struct __pyx_obj_4cdec_5_cdec_SufficientStats *__pyx_f_4cdec_5_cdec_as_stats(PyObject *, PyObject *); /*proto*/
 static float __pyx_f_4cdec_5_cdec__compute_score(void *, SufficientStats *); /*proto*/
 static void __pyx_f_4cdec_5_cdec__compute_sufficient_stats(void *, std::string *, std::vector<std::string>  *, SufficientStats *); /*proto*/
-static std::string __pyx_convert_string_from_py_(PyObject *); /*proto*/
+static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/
 #define __Pyx_MODULE_NAME "cdec._cdec"
 int __pyx_module_is_main_cdec___cdec = 0;
 
@@ -2440,11 +2546,9 @@ static char __pyx_k_Hypergraph_sample_hypotheses[] = "Hypergraph.sample_hypothes
 static char __pyx_k_cannot_create_lattice_from_s[] = "cannot create lattice from %s";
 static char __pyx_k_Cannot_translate_input_type_s[] = "Cannot translate input type %s";
 static char __pyx_k_Hypergraph_unique_kbest_trees[] = "Hypergraph.unique_kbest_trees";
-static char __pyx_k_Users_waziz_workspace_mtm14_my[] = "/Users/waziz/workspace/mtm14/my-cdec/python/cdec/lattice.pxi";
 static char __pyx_k_cannot_reweight_hypergraph_with[] = "cannot reweight hypergraph with %s";
+static char __pyx_k_home_pks_src_cdec_dtrain_python[] = "/home/pks/src/cdec-dtrain/python/cdec/lattice.pxi";
 static char __pyx_k_Hypergraph_unique_kbest_features[] = "Hypergraph.unique_kbest_features";
-static char __pyx_k_Users_waziz_workspace_mtm14_my_2[] = "/Users/waziz/workspace/mtm14/my-cdec/python/cdec/grammar.pxi";
-static char __pyx_k_Users_waziz_workspace_mtm14_my_3[] = "/Users/waziz/workspace/mtm14/my-cdec/python/cdec/_cdec.pyx";
 static char __pyx_k_candidate_set_index_out_of_range[] = "candidate set index out of range";
 static char __pyx_k_cannot_initialize_weights_with_s[] = "cannot initialize weights with %s";
 static char __pyx_k_cannot_intersect_hypergraph_with[] = "cannot intersect hypergraph with %s";
@@ -2453,6 +2557,8 @@ static char __pyx_k_comparison_not_implemented_for_H[] = "comparison not impleme
 static char __pyx_k_comparison_not_implemented_for_S[] = "comparison not implemented for SparseVector";
 static char __pyx_k_sufficient_stats_vector_index_ou[] = "sufficient stats vector index out of range";
 static char __pyx_k_the_grammar_should_contain_TRule[] = "the grammar should contain TRule objects";
+static char __pyx_k_home_pks_src_cdec_dtrain_python_2[] = "/home/pks/src/cdec-dtrain/python/cdec/grammar.pxi";
+static char __pyx_k_home_pks_src_cdec_dtrain_python_3[] = "/home/pks/src/cdec-dtrain/python/cdec/_cdec.pyx";
 static char __pyx_k_comparison_not_implemented_for_H_2[] = "comparison not implemented for HypergraphNode";
 static PyObject *__pyx_n_s_BLEU;
 static PyObject *__pyx_n_s_CER;
@@ -2488,9 +2594,6 @@ static PyObject *__pyx_n_s_SufficientStats___iter;
 static PyObject *__pyx_n_s_TER;
 static PyObject *__pyx_n_s_TRule___get;
 static PyObject *__pyx_n_s_TypeError;
-static PyObject *__pyx_kp_s_Users_waziz_workspace_mtm14_my;
-static PyObject *__pyx_kp_s_Users_waziz_workspace_mtm14_my_2;
-static PyObject *__pyx_kp_s_Users_waziz_workspace_mtm14_my_3;
 static PyObject *__pyx_n_s_ValueError;
 static PyObject *__pyx_kp_s__10;
 static PyObject *__pyx_kp_s__11;
@@ -2548,6 +2651,9 @@ static PyObject *__pyx_n_s_genexpr;
 static PyObject *__pyx_n_s_get;
 static PyObject *__pyx_n_s_get_2;
 static PyObject *__pyx_n_s_grammar;
+static PyObject *__pyx_kp_s_home_pks_src_cdec_dtrain_python;
+static PyObject *__pyx_kp_s_home_pks_src_cdec_dtrain_python_2;
+static PyObject *__pyx_kp_s_home_pks_src_cdec_dtrain_python_3;
 static PyObject *__pyx_n_s_hyp;
 static PyObject *__pyx_n_s_hypergraph;
 static PyObject *__pyx_n_s_i;
@@ -3250,7 +3356,12 @@ static int __pyx_pf_4cdec_5_cdec_11DenseVector_8__setitem__(struct __pyx_obj_4cd
  *         self.vector[0][fid] = value
  * 
  */
-    __pyx_v_self->vector->resize((__pyx_v_fid + 1));
+    try {
+      __pyx_v_self->vector->resize((__pyx_v_fid + 1));
+    } catch(...) {
+      __Pyx_CppExn2PyErr();
+      {__pyx_filename = __pyx_f[1]; __pyx_lineno = 29; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    }
     goto __pyx_L4;
   }
   __pyx_L4:;
@@ -6636,16 +6747,11 @@ static struct __pyx_obj_4cdec_5_cdec_TRule *__pyx_f_4cdec_5_cdec_convert_rule(st
   }
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
-  __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PySequence_List(__pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
-  PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1);
-  __Pyx_GIVEREF(__pyx_t_1);
-  __pyx_t_1 = 0;
-  __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyList_Type))), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_t_1);
-  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
-  __pyx_v_a = ((PyObject*)__pyx_t_1);
-  __pyx_t_1 = 0;
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __pyx_v_a = ((PyObject*)__pyx_t_2);
+  __pyx_t_2 = 0;
 
   /* "cdec/grammar.pxi":47
  *             e.append(_sa.sym_tostring(esyms[i]))
@@ -6655,30 +6761,30 @@ static struct __pyx_obj_4cdec_5_cdec_TRule *__pyx_f_4cdec_5_cdec_convert_rule(st
  * cdef class TRule:
  */
   __Pyx_XDECREF(((PyObject *)__pyx_r));
-  __pyx_t_1 = __Pyx_PyBytes_FromString(__pyx_v_lhs); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_t_1);
-  __pyx_t_2 = PyTuple_New(5); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __Pyx_PyBytes_FromString(__pyx_v_lhs); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
-  PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1);
-  __Pyx_GIVEREF(__pyx_t_1);
+  __pyx_t_1 = PyTuple_New(5); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_1);
+  PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_2);
+  __Pyx_GIVEREF(__pyx_t_2);
   __Pyx_INCREF(__pyx_v_f);
-  PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_f);
+  PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_f);
   __Pyx_GIVEREF(__pyx_v_f);
   __Pyx_INCREF(__pyx_v_e);
-  PyTuple_SET_ITEM(__pyx_t_2, 2, __pyx_v_e);
+  PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_v_e);
   __Pyx_GIVEREF(__pyx_v_e);
   __Pyx_INCREF(__pyx_v_scores);
-  PyTuple_SET_ITEM(__pyx_t_2, 3, __pyx_v_scores);
+  PyTuple_SET_ITEM(__pyx_t_1, 3, __pyx_v_scores);
   __Pyx_GIVEREF(__pyx_v_scores);
   __Pyx_INCREF(__pyx_v_a);
-  PyTuple_SET_ITEM(__pyx_t_2, 4, __pyx_v_a);
+  PyTuple_SET_ITEM(__pyx_t_1, 4, __pyx_v_a);
   __Pyx_GIVEREF(__pyx_v_a);
-  __pyx_t_1 = 0;
-  __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_4cdec_5_cdec_TRule)), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_t_1);
-  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
-  __pyx_r = ((struct __pyx_obj_4cdec_5_cdec_TRule *)__pyx_t_1);
-  __pyx_t_1 = 0;
+  __pyx_t_2 = 0;
+  __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)__pyx_ptype_4cdec_5_cdec_TRule)), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_2);
+  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+  __pyx_r = ((struct __pyx_obj_4cdec_5_cdec_TRule *)__pyx_t_2);
+  __pyx_t_2 = 0;
   goto __pyx_L0;
 
   /* "cdec/grammar.pxi":30
@@ -6972,7 +7078,7 @@ static int __pyx_pf_4cdec_5_cdec_5TRule___init__(struct __pyx_obj_4cdec_5_cdec_T
  *         self.rule.get().ComputeArity()
  * 
  */
-    __pyx_t_3 = __pyx_convert_string_from_py_(__pyx_v_text); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_v_text); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __pyx_v_self->rule->get()->ReadFromString(__pyx_t_3, 0);
     goto __pyx_L8;
   }
@@ -7369,7 +7475,12 @@ static int __pyx_pf_4cdec_5_cdec_5TRule_1f_2__set__(struct __pyx_obj_4cdec_5_cde
  *             cdef int idx = 0
  */
   __pyx_t_1 = PyObject_Length(__pyx_v_f); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_v_f_->resize(__pyx_t_1);
+  try {
+    __pyx_v_f_->resize(__pyx_t_1);
+  } catch(...) {
+    __Pyx_CppExn2PyErr();
+    {__pyx_filename = __pyx_f[2]; __pyx_lineno = 99; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
 
   /* "cdec/grammar.pxi":101
  *             f_.resize(len(f))
@@ -7722,7 +7833,12 @@ static int __pyx_pf_4cdec_5_cdec_5TRule_1e_2__set__(struct __pyx_obj_4cdec_5_cde
  *             for i in range(len(e)):
  */
   __pyx_t_1 = PyObject_Length(__pyx_v_e); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_v_e_->resize(__pyx_t_1);
+  try {
+    __pyx_v_e_->resize(__pyx_t_1);
+  } catch(...) {
+    __Pyx_CppExn2PyErr();
+    {__pyx_filename = __pyx_f[2]; __pyx_lineno = 127; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
 
   /* "cdec/grammar.pxi":129
  *             e_.resize(len(e))
@@ -8038,7 +8154,12 @@ static int __pyx_pf_4cdec_5_cdec_5TRule_1a_3__set__(struct __pyx_obj_4cdec_5_cde
  *             cdef int s, t
  */
   __pyx_t_1 = PyObject_Length(__pyx_v_a); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_v_a_->resize(__pyx_t_1);
+  try {
+    __pyx_v_a_->resize(__pyx_t_1);
+  } catch(...) {
+    __Pyx_CppExn2PyErr();
+    {__pyx_filename = __pyx_f[2]; __pyx_lineno = 145; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
 
   /* "cdec/grammar.pxi":148
  *             cdef unsigned i
@@ -9692,7 +9813,7 @@ static int __pyx_pf_4cdec_5_cdec_7Grammar_4name_2__set__(struct __pyx_obj_4cdec_
  * 
  * cdef class TextGrammar(Grammar):
  */
-  __pyx_t_2 = __pyx_convert_string_from_py_(__pyx_v_name); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 221; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_v_name); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 221; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_v_self->grammar->get()->SetGrammarName(__pyx_t_2);
 
   /* "cdec/grammar.pxi":219
@@ -10958,12 +11079,10 @@ static PyObject *__pyx_gb_4cdec_5_cdec_10Hypergraph_17generator5(__pyx_Generator
  */
       __pyx_t_4 = ((!(__pyx_cur_scope->__pyx_v_f_derivation != 0)) != 0);
       if (!__pyx_t_4) {
-        goto __pyx_L11_next_or;
       } else {
         __pyx_t_3 = __pyx_t_4;
         goto __pyx_L10_bool_binop_done;
       }
-      __pyx_L11_next_or:;
       __pyx_t_4 = ((!(__pyx_cur_scope->__pyx_v_e_derivation != 0)) != 0);
       __pyx_t_3 = __pyx_t_4;
       __pyx_L10_bool_binop_done:;
@@ -11817,12 +11936,10 @@ static PyObject *__pyx_gb_4cdec_5_cdec_10Hypergraph_26generator8(__pyx_Generator
  */
       __pyx_t_4 = ((!(__pyx_cur_scope->__pyx_v_f_derivation != 0)) != 0);
       if (!__pyx_t_4) {
-        goto __pyx_L11_next_or;
       } else {
         __pyx_t_3 = __pyx_t_4;
         goto __pyx_L10_bool_binop_done;
       }
-      __pyx_L11_next_or:;
       __pyx_t_4 = ((!(__pyx_cur_scope->__pyx_v_e_derivation != 0)) != 0);
       __pyx_t_3 = __pyx_t_4;
       __pyx_L10_bool_binop_done:;
@@ -15893,7 +16010,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_14HypergraphNode___richcmp__(struct __pyx
   return __pyx_r;
 }
 
-/* "cdec/lattice.pxi":6
+/* "cdec/lattice.pxi":7
  *     cdef lattice.Lattice* lattice
  * 
  *     def __cinit__(self):             # <<<<<<<<<<<<<<
@@ -15922,7 +16039,7 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice___cinit__(struct __pyx_obj_4cdec_5_cde
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__cinit__", 0);
 
-  /* "cdec/lattice.pxi":7
+  /* "cdec/lattice.pxi":8
  * 
  *     def __cinit__(self):
  *         self.lattice = new lattice.Lattice()             # <<<<<<<<<<<<<<
@@ -15931,7 +16048,7 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice___cinit__(struct __pyx_obj_4cdec_5_cde
  */
   __pyx_v_self->lattice = new Lattice();
 
-  /* "cdec/lattice.pxi":6
+  /* "cdec/lattice.pxi":7
  *     cdef lattice.Lattice* lattice
  * 
  *     def __cinit__(self):             # <<<<<<<<<<<<<<
@@ -15945,7 +16062,7 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice___cinit__(struct __pyx_obj_4cdec_5_cde
   return __pyx_r;
 }
 
-/* "cdec/lattice.pxi":9
+/* "cdec/lattice.pxi":10
  *         self.lattice = new lattice.Lattice()
  * 
  *     def __init__(self, inp):             # <<<<<<<<<<<<<<
@@ -15985,7 +16102,7 @@ static int __pyx_pw_4cdec_5_cdec_7Lattice_3__init__(PyObject *__pyx_v_self, PyOb
         else goto __pyx_L5_argtuple_error;
       }
       if (unlikely(kw_args > 0)) {
-        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+        if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
       }
     } else if (PyTuple_GET_SIZE(__pyx_args) != 1) {
       goto __pyx_L5_argtuple_error;
@@ -15996,7 +16113,7 @@ static int __pyx_pw_4cdec_5_cdec_7Lattice_3__init__(PyObject *__pyx_v_self, PyOb
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L5_argtuple_error:;
-  __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 9; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+  __Pyx_RaiseArgtupleInvalid("__init__", 1, 1, 1, PyTuple_GET_SIZE(__pyx_args)); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 10; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   __pyx_L3_error:;
   __Pyx_AddTraceback("cdec._cdec.Lattice.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __Pyx_RefNannyFinishContext();
@@ -16027,7 +16144,7 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_2__init__(struct __pyx_obj_4cdec_5_cde
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__init__", 0);
 
-  /* "cdec/lattice.pxi":12
+  /* "cdec/lattice.pxi":13
  *         """Lattice(tuple) -> Lattice from node list.
  *         Lattice(string) -> Lattice from PLF representation."""
  *         if isinstance(inp, tuple):             # <<<<<<<<<<<<<<
@@ -16038,17 +16155,17 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_2__init__(struct __pyx_obj_4cdec_5_cde
   __pyx_t_2 = (__pyx_t_1 != 0);
   if (__pyx_t_2) {
 
-    /* "cdec/lattice.pxi":13
+    /* "cdec/lattice.pxi":14
  *         Lattice(string) -> Lattice from PLF representation."""
  *         if isinstance(inp, tuple):
  *             self.lattice.resize(len(inp))             # <<<<<<<<<<<<<<
  *             for i, arcs in enumerate(inp):
  *                 self[i] = arcs
  */
-    __pyx_t_3 = PyObject_Length(__pyx_v_inp); if (unlikely(__pyx_t_3 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 13; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_3 = PyObject_Length(__pyx_v_inp); if (unlikely(__pyx_t_3 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __pyx_v_self->lattice->resize(__pyx_t_3);
 
-    /* "cdec/lattice.pxi":14
+    /* "cdec/lattice.pxi":15
  *         if isinstance(inp, tuple):
  *             self.lattice.resize(len(inp))
  *             for i, arcs in enumerate(inp):             # <<<<<<<<<<<<<<
@@ -16061,25 +16178,25 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_2__init__(struct __pyx_obj_4cdec_5_cde
       __pyx_t_5 = __pyx_v_inp; __Pyx_INCREF(__pyx_t_5); __pyx_t_3 = 0;
       __pyx_t_6 = NULL;
     } else {
-      __pyx_t_3 = -1; __pyx_t_5 = PyObject_GetIter(__pyx_v_inp); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_3 = -1; __pyx_t_5 = PyObject_GetIter(__pyx_v_inp); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_5);
-      __pyx_t_6 = Py_TYPE(__pyx_t_5)->tp_iternext; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_6 = Py_TYPE(__pyx_t_5)->tp_iternext; if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     }
     for (;;) {
       if (likely(!__pyx_t_6)) {
         if (likely(PyList_CheckExact(__pyx_t_5))) {
           if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_5)) break;
           #if CYTHON_COMPILING_IN_CPYTHON
-          __pyx_t_7 = PyList_GET_ITEM(__pyx_t_5, __pyx_t_3); __Pyx_INCREF(__pyx_t_7); __pyx_t_3++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_7 = PyList_GET_ITEM(__pyx_t_5, __pyx_t_3); __Pyx_INCREF(__pyx_t_7); __pyx_t_3++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           #else
-          __pyx_t_7 = PySequence_ITEM(__pyx_t_5, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_7 = PySequence_ITEM(__pyx_t_5, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           #endif
         } else {
           if (__pyx_t_3 >= PyTuple_GET_SIZE(__pyx_t_5)) break;
           #if CYTHON_COMPILING_IN_CPYTHON
-          __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_5, __pyx_t_3); __Pyx_INCREF(__pyx_t_7); __pyx_t_3++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_7 = PyTuple_GET_ITEM(__pyx_t_5, __pyx_t_3); __Pyx_INCREF(__pyx_t_7); __pyx_t_3++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           #else
-          __pyx_t_7 = PySequence_ITEM(__pyx_t_5, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_7 = PySequence_ITEM(__pyx_t_5, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           #endif
         }
       } else {
@@ -16088,7 +16205,7 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_2__init__(struct __pyx_obj_4cdec_5_cde
           PyObject* exc_type = PyErr_Occurred();
           if (exc_type) {
             if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
-            else {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+            else {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           }
           break;
         }
@@ -16098,22 +16215,22 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_2__init__(struct __pyx_obj_4cdec_5_cde
       __pyx_t_7 = 0;
       __Pyx_INCREF(__pyx_t_4);
       __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_4);
-      __pyx_t_7 = PyNumber_Add(__pyx_t_4, __pyx_int_1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_7 = PyNumber_Add(__pyx_t_4, __pyx_int_1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_7);
       __Pyx_DECREF(__pyx_t_4);
       __pyx_t_4 = __pyx_t_7;
       __pyx_t_7 = 0;
 
-      /* "cdec/lattice.pxi":15
+      /* "cdec/lattice.pxi":16
  *             self.lattice.resize(len(inp))
  *             for i, arcs in enumerate(inp):
  *                 self[i] = arcs             # <<<<<<<<<<<<<<
  *         elif isinstance(inp, basestring):
  *             lattice.ConvertTextOrPLF(as_str(inp), self.lattice)
  */
-      if (unlikely(PyObject_SetItem(((PyObject *)__pyx_v_self), __pyx_v_i, __pyx_v_arcs) < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      if (unlikely(PyObject_SetItem(((PyObject *)__pyx_v_self), __pyx_v_i, __pyx_v_arcs) < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 16; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-      /* "cdec/lattice.pxi":14
+      /* "cdec/lattice.pxi":15
  *         if isinstance(inp, tuple):
  *             self.lattice.resize(len(inp))
  *             for i, arcs in enumerate(inp):             # <<<<<<<<<<<<<<
@@ -16126,7 +16243,7 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_2__init__(struct __pyx_obj_4cdec_5_cde
     goto __pyx_L3;
   }
 
-  /* "cdec/lattice.pxi":16
+  /* "cdec/lattice.pxi":17
  *             for i, arcs in enumerate(inp):
  *                 self[i] = arcs
  *         elif isinstance(inp, basestring):             # <<<<<<<<<<<<<<
@@ -16137,46 +16254,46 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_2__init__(struct __pyx_obj_4cdec_5_cde
   __pyx_t_1 = (__pyx_t_2 != 0);
   if (__pyx_t_1) {
 
-    /* "cdec/lattice.pxi":17
+    /* "cdec/lattice.pxi":18
  *                 self[i] = arcs
  *         elif isinstance(inp, basestring):
  *             lattice.ConvertTextOrPLF(as_str(inp), self.lattice)             # <<<<<<<<<<<<<<
  *         else:
  *             raise TypeError('cannot create lattice from %s' % type(inp))
  */
-    __pyx_t_4 = __pyx_f_4cdec_5_cdec_as_str(__pyx_v_inp, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = __pyx_f_4cdec_5_cdec_as_str(__pyx_v_inp, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
-    __pyx_t_8 = __pyx_convert_string_from_py_(__pyx_t_4); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_8 = __pyx_convert_string_from_py_std__in_string(__pyx_t_4); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
     LatticeTools::ConvertTextOrPLF(__pyx_t_8, __pyx_v_self->lattice);
     goto __pyx_L3;
   }
   /*else*/ {
 
-    /* "cdec/lattice.pxi":19
+    /* "cdec/lattice.pxi":20
  *             lattice.ConvertTextOrPLF(as_str(inp), self.lattice)
  *         else:
  *             raise TypeError('cannot create lattice from %s' % type(inp))             # <<<<<<<<<<<<<<
  * 
  *     def __dealloc__(self):
  */
-    __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_cannot_create_lattice_from_s, ((PyObject *)Py_TYPE(__pyx_v_inp))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_cannot_create_lattice_from_s, ((PyObject *)Py_TYPE(__pyx_v_inp))); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
-    __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_5 = PyTuple_New(1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_5);
     PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4);
     __Pyx_GIVEREF(__pyx_t_4);
     __pyx_t_4 = 0;
-    __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
     __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
     __Pyx_Raise(__pyx_t_4, 0, 0, 0);
     __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
-    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 19; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 20; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
   __pyx_L3:;
 
-  /* "cdec/lattice.pxi":9
+  /* "cdec/lattice.pxi":10
  *         self.lattice = new lattice.Lattice()
  * 
  *     def __init__(self, inp):             # <<<<<<<<<<<<<<
@@ -16200,7 +16317,7 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_2__init__(struct __pyx_obj_4cdec_5_cde
   return __pyx_r;
 }
 
-/* "cdec/lattice.pxi":21
+/* "cdec/lattice.pxi":22
  *             raise TypeError('cannot create lattice from %s' % type(inp))
  * 
  *     def __dealloc__(self):             # <<<<<<<<<<<<<<
@@ -16223,7 +16340,7 @@ static void __pyx_pf_4cdec_5_cdec_7Lattice_4__dealloc__(struct __pyx_obj_4cdec_5
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__dealloc__", 0);
 
-  /* "cdec/lattice.pxi":22
+  /* "cdec/lattice.pxi":23
  * 
  *     def __dealloc__(self):
  *         del self.lattice             # <<<<<<<<<<<<<<
@@ -16232,7 +16349,7 @@ static void __pyx_pf_4cdec_5_cdec_7Lattice_4__dealloc__(struct __pyx_obj_4cdec_5
  */
   delete __pyx_v_self->lattice;
 
-  /* "cdec/lattice.pxi":21
+  /* "cdec/lattice.pxi":22
  *             raise TypeError('cannot create lattice from %s' % type(inp))
  * 
  *     def __dealloc__(self):             # <<<<<<<<<<<<<<
@@ -16244,7 +16361,7 @@ static void __pyx_pf_4cdec_5_cdec_7Lattice_4__dealloc__(struct __pyx_obj_4cdec_5
   __Pyx_RefNannyFinishContext();
 }
 
-/* "cdec/lattice.pxi":24
+/* "cdec/lattice.pxi":25
  *         del self.lattice
  * 
  *     def __getitem__(self, int index):             # <<<<<<<<<<<<<<
@@ -16263,7 +16380,7 @@ static PyObject *__pyx_pw_4cdec_5_cdec_7Lattice_7__getitem__(PyObject *__pyx_v_s
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__getitem__ (wrapper)", 0);
   assert(__pyx_arg_index); {
-    __pyx_v_index = __Pyx_PyInt_As_int(__pyx_arg_index); if (unlikely((__pyx_v_index == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 24; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    __pyx_v_index = __Pyx_PyInt_As_int(__pyx_arg_index); if (unlikely((__pyx_v_index == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 25; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L3_error:;
@@ -16293,14 +16410,13 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_6__getitem__(struct __pyx_obj_4c
   size_t __pyx_t_5;
   unsigned int __pyx_t_6;
   PyObject *__pyx_t_7 = NULL;
-  PyObject *__pyx_t_8 = NULL;
-  int __pyx_t_9;
+  int __pyx_t_8;
   int __pyx_lineno = 0;
   const char *__pyx_filename = NULL;
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__getitem__", 0);
 
-  /* "cdec/lattice.pxi":25
+  /* "cdec/lattice.pxi":26
  * 
  *     def __getitem__(self, int index):
  *         if not 0 <= index < len(self):             # <<<<<<<<<<<<<<
@@ -16309,39 +16425,39 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_6__getitem__(struct __pyx_obj_4c
  */
   __pyx_t_1 = (0 <= __pyx_v_index);
   if (__pyx_t_1) {
-    __pyx_t_2 = PyObject_Length(((PyObject *)__pyx_v_self)); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 25; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_2 = PyObject_Length(((PyObject *)__pyx_v_self)); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __pyx_t_1 = (__pyx_v_index < __pyx_t_2);
   }
   __pyx_t_3 = ((!(__pyx_t_1 != 0)) != 0);
   if (__pyx_t_3) {
 
-    /* "cdec/lattice.pxi":26
+    /* "cdec/lattice.pxi":27
  *     def __getitem__(self, int index):
  *         if not 0 <= index < len(self):
  *             raise IndexError('lattice index out of range')             # <<<<<<<<<<<<<<
  *         arcs = []
  *         cdef vector[lattice.LatticeArc] arc_vector = self.lattice[0][index]
  */
-    __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IndexError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IndexError, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
     __Pyx_Raise(__pyx_t_4, 0, 0, 0);
     __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
-    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
 
-  /* "cdec/lattice.pxi":27
+  /* "cdec/lattice.pxi":28
  *         if not 0 <= index < len(self):
  *             raise IndexError('lattice index out of range')
  *         arcs = []             # <<<<<<<<<<<<<<
  *         cdef vector[lattice.LatticeArc] arc_vector = self.lattice[0][index]
  *         cdef lattice.LatticeArc* arc
  */
-  __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_4);
   __pyx_v_arcs = ((PyObject*)__pyx_t_4);
   __pyx_t_4 = 0;
 
-  /* "cdec/lattice.pxi":28
+  /* "cdec/lattice.pxi":29
  *             raise IndexError('lattice index out of range')
  *         arcs = []
  *         cdef vector[lattice.LatticeArc] arc_vector = self.lattice[0][index]             # <<<<<<<<<<<<<<
@@ -16350,7 +16466,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_6__getitem__(struct __pyx_obj_4c
  */
   __pyx_v_arc_vector = ((__pyx_v_self->lattice[0])[__pyx_v_index]);
 
-  /* "cdec/lattice.pxi":31
+  /* "cdec/lattice.pxi":32
  *         cdef lattice.LatticeArc* arc
  *         cdef unsigned i
  *         for i in range(arc_vector.size()):             # <<<<<<<<<<<<<<
@@ -16361,25 +16477,25 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_6__getitem__(struct __pyx_obj_4c
   for (__pyx_t_6 = 0; __pyx_t_6 < __pyx_t_5; __pyx_t_6+=1) {
     __pyx_v_i = __pyx_t_6;
 
-    /* "cdec/lattice.pxi":32
+    /* "cdec/lattice.pxi":33
  *         cdef unsigned i
  *         for i in range(arc_vector.size()):
  *             arc = &arc_vector[i]             # <<<<<<<<<<<<<<
  *             label = unicode(TDConvert(arc.label).c_str(), 'utf8')
- *             arcs.append((label, arc.cost, arc.dist2next))
+ *             #arcs.append((label, arc.features, arc.dist2next))
  */
     __pyx_v_arc = (&(__pyx_v_arc_vector[__pyx_v_i]));
 
-    /* "cdec/lattice.pxi":33
+    /* "cdec/lattice.pxi":34
  *         for i in range(arc_vector.size()):
  *             arc = &arc_vector[i]
  *             label = unicode(TDConvert(arc.label).c_str(), 'utf8')             # <<<<<<<<<<<<<<
- *             arcs.append((label, arc.cost, arc.dist2next))
- *         return tuple(arcs)
+ *             #arcs.append((label, arc.features, arc.dist2next))
+ *             arcs.append((label, arc.dist2next))
  */
-    __pyx_t_4 = __Pyx_PyBytes_FromString(TD::Convert(__pyx_v_arc->label).c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = __Pyx_PyBytes_FromString(TD::Convert(__pyx_v_arc->label).c_str()); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
-    __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_7);
     PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_4);
     __Pyx_GIVEREF(__pyx_t_4);
@@ -16387,53 +16503,48 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_6__getitem__(struct __pyx_obj_4c
     PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_n_s_utf8);
     __Pyx_GIVEREF(__pyx_n_s_utf8);
     __pyx_t_4 = 0;
-    __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyUnicode_Type))), __pyx_t_7, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 33; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyUnicode_Type))), __pyx_t_7, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
     __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
     __Pyx_XDECREF_SET(__pyx_v_label, ((PyObject*)__pyx_t_4));
     __pyx_t_4 = 0;
 
-    /* "cdec/lattice.pxi":34
- *             arc = &arc_vector[i]
+    /* "cdec/lattice.pxi":36
  *             label = unicode(TDConvert(arc.label).c_str(), 'utf8')
- *             arcs.append((label, arc.cost, arc.dist2next))             # <<<<<<<<<<<<<<
+ *             #arcs.append((label, arc.features, arc.dist2next))
+ *             arcs.append((label, arc.dist2next))             # <<<<<<<<<<<<<<
  *         return tuple(arcs)
  * 
  */
-    __pyx_t_4 = PyFloat_FromDouble(__pyx_v_arc->cost); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = __Pyx_PyInt_From_int(__pyx_v_arc->dist2next); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
-    __pyx_t_7 = __Pyx_PyInt_From_int(__pyx_v_arc->dist2next); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_7);
-    __pyx_t_8 = PyTuple_New(3); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __Pyx_GOTREF(__pyx_t_8);
     __Pyx_INCREF(__pyx_v_label);
-    PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_label);
+    PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_v_label);
     __Pyx_GIVEREF(__pyx_v_label);
-    PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_t_4);
+    PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_t_4);
     __Pyx_GIVEREF(__pyx_t_4);
-    PyTuple_SET_ITEM(__pyx_t_8, 2, __pyx_t_7);
-    __Pyx_GIVEREF(__pyx_t_7);
     __pyx_t_4 = 0;
-    __pyx_t_7 = 0;
-    __pyx_t_9 = __Pyx_PyList_Append(__pyx_v_arcs, __pyx_t_8); if (unlikely(__pyx_t_9 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 34; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+    __pyx_t_8 = __Pyx_PyList_Append(__pyx_v_arcs, __pyx_t_7); if (unlikely(__pyx_t_8 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 36; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
   }
 
-  /* "cdec/lattice.pxi":35
- *             label = unicode(TDConvert(arc.label).c_str(), 'utf8')
- *             arcs.append((label, arc.cost, arc.dist2next))
+  /* "cdec/lattice.pxi":37
+ *             #arcs.append((label, arc.features, arc.dist2next))
+ *             arcs.append((label, arc.dist2next))
  *         return tuple(arcs)             # <<<<<<<<<<<<<<
  * 
  *     def __setitem__(self, int index, tuple arcs):
  */
   __Pyx_XDECREF(__pyx_r);
-  __pyx_t_8 = PyList_AsTuple(__pyx_v_arcs); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 35; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __Pyx_GOTREF(__pyx_t_8);
-  __pyx_r = __pyx_t_8;
-  __pyx_t_8 = 0;
+  __pyx_t_7 = PyList_AsTuple(__pyx_v_arcs); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __Pyx_GOTREF(__pyx_t_7);
+  __pyx_r = __pyx_t_7;
+  __pyx_t_7 = 0;
   goto __pyx_L0;
 
-  /* "cdec/lattice.pxi":24
+  /* "cdec/lattice.pxi":25
  *         del self.lattice
  * 
  *     def __getitem__(self, int index):             # <<<<<<<<<<<<<<
@@ -16445,7 +16556,6 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_6__getitem__(struct __pyx_obj_4c
   __pyx_L1_error:;
   __Pyx_XDECREF(__pyx_t_4);
   __Pyx_XDECREF(__pyx_t_7);
-  __Pyx_XDECREF(__pyx_t_8);
   __Pyx_AddTraceback("cdec._cdec.Lattice.__getitem__", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __pyx_r = NULL;
   __pyx_L0:;
@@ -16456,7 +16566,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_6__getitem__(struct __pyx_obj_4c
   return __pyx_r;
 }
 
-/* "cdec/lattice.pxi":37
+/* "cdec/lattice.pxi":39
  *         return tuple(arcs)
  * 
  *     def __setitem__(self, int index, tuple arcs):             # <<<<<<<<<<<<<<
@@ -16475,7 +16585,7 @@ static int __pyx_pw_4cdec_5_cdec_7Lattice_9__setitem__(PyObject *__pyx_v_self, P
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__setitem__ (wrapper)", 0);
   assert(__pyx_arg_index); {
-    __pyx_v_index = __Pyx_PyInt_As_int(__pyx_arg_index); if (unlikely((__pyx_v_index == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
+    __pyx_v_index = __Pyx_PyInt_As_int(__pyx_arg_index); if (unlikely((__pyx_v_index == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L3_error;}
   }
   goto __pyx_L4_argument_unpacking_done;
   __pyx_L3_error:;
@@ -16483,7 +16593,7 @@ static int __pyx_pw_4cdec_5_cdec_7Lattice_9__setitem__(PyObject *__pyx_v_self, P
   __Pyx_RefNannyFinishContext();
   return -1;
   __pyx_L4_argument_unpacking_done:;
-  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_arcs), (&PyTuple_Type), 1, "arcs", 1))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 37; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_arcs), (&PyTuple_Type), 1, "arcs", 1))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_r = __pyx_pf_4cdec_5_cdec_7Lattice_8__setitem__(((struct __pyx_obj_4cdec_5_cdec_Lattice *)__pyx_v_self), ((int)__pyx_v_index), ((PyObject*)__pyx_v_arcs));
 
   /* function exit code */
@@ -16498,9 +16608,8 @@ static int __pyx_pw_4cdec_5_cdec_7Lattice_9__setitem__(PyObject *__pyx_v_self, P
 static int __pyx_pf_4cdec_5_cdec_7Lattice_8__setitem__(struct __pyx_obj_4cdec_5_cdec_Lattice *__pyx_v_self, int __pyx_v_index, PyObject *__pyx_v_arcs) {
   LatticeArc *__pyx_v_arc;
   PyObject *__pyx_v_label = NULL;
-  PyObject *__pyx_v_cost = NULL;
-  PyObject *__pyx_v_dist2next = NULL;
-  PyObject *__pyx_v_label_str = NULL;
+  CYTHON_UNUSED PyObject *__pyx_v_dist2next = NULL;
+  CYTHON_UNUSED PyObject *__pyx_v_label_str = NULL;
   int __pyx_r;
   __Pyx_RefNannyDeclarations
   int __pyx_t_1;
@@ -16511,17 +16620,13 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_8__setitem__(struct __pyx_obj_4cdec_5_
   PyObject *__pyx_t_6 = NULL;
   PyObject *__pyx_t_7 = NULL;
   PyObject *__pyx_t_8 = NULL;
-  PyObject *__pyx_t_9 = NULL;
-  PyObject *(*__pyx_t_10)(PyObject *);
-  char *__pyx_t_11;
-  double __pyx_t_12;
-  int __pyx_t_13;
+  PyObject *(*__pyx_t_9)(PyObject *);
   int __pyx_lineno = 0;
   const char *__pyx_filename = NULL;
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__setitem__", 0);
 
-  /* "cdec/lattice.pxi":38
+  /* "cdec/lattice.pxi":40
  * 
  *     def __setitem__(self, int index, tuple arcs):
  *         if not 0 <= index < len(self):             # <<<<<<<<<<<<<<
@@ -16530,44 +16635,44 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_8__setitem__(struct __pyx_obj_4cdec_5_
  */
   __pyx_t_1 = (0 <= __pyx_v_index);
   if (__pyx_t_1) {
-    __pyx_t_2 = PyObject_Length(((PyObject *)__pyx_v_self)); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 38; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_2 = PyObject_Length(((PyObject *)__pyx_v_self)); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 40; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __pyx_t_1 = (__pyx_v_index < __pyx_t_2);
   }
   __pyx_t_3 = ((!(__pyx_t_1 != 0)) != 0);
   if (__pyx_t_3) {
 
-    /* "cdec/lattice.pxi":39
+    /* "cdec/lattice.pxi":41
  *     def __setitem__(self, int index, tuple arcs):
  *         if not 0 <= index < len(self):
  *             raise IndexError('lattice index out of range')             # <<<<<<<<<<<<<<
  *         cdef lattice.LatticeArc* arc
- *         for (label, cost, dist2next) in arcs:
+ *         for (label, dist2next) in arcs:
  */
-    __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IndexError, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = __Pyx_PyObject_Call(__pyx_builtin_IndexError, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_4);
     __Pyx_Raise(__pyx_t_4, 0, 0, 0);
     __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
-    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
 
-  /* "cdec/lattice.pxi":41
+  /* "cdec/lattice.pxi":43
  *             raise IndexError('lattice index out of range')
  *         cdef lattice.LatticeArc* arc
- *         for (label, cost, dist2next) in arcs:             # <<<<<<<<<<<<<<
+ *         for (label, dist2next) in arcs:             # <<<<<<<<<<<<<<
  *             label_str = as_str(label)
- *             arc = new lattice.LatticeArc(TDConvert(label_str), cost, dist2next)
+ *             arc = new lattice.LatticeArc() #TDConvert(label_str), features, dist2next)
  */
   if (unlikely(__pyx_v_arcs == Py_None)) {
     PyErr_SetString(PyExc_TypeError, "'NoneType' object is not iterable");
-    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
   __pyx_t_4 = __pyx_v_arcs; __Pyx_INCREF(__pyx_t_4); __pyx_t_2 = 0;
   for (;;) {
     if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_4)) break;
     #if CYTHON_COMPILING_IN_CPYTHON
-    __pyx_t_5 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_5); __pyx_t_2++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_5 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_2); __Pyx_INCREF(__pyx_t_5); __pyx_t_2++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     #else
-    __pyx_t_5 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_5 = PySequence_ITEM(__pyx_t_4, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     #endif
     if ((likely(PyTuple_CheckExact(__pyx_t_5))) || (PyList_CheckExact(__pyx_t_5))) {
       PyObject* sequence = __pyx_t_5;
@@ -16576,98 +16681,91 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_8__setitem__(struct __pyx_obj_4cdec_5_
       #else
       Py_ssize_t size = PySequence_Size(sequence);
       #endif
-      if (unlikely(size != 3)) {
-        if (size > 3) __Pyx_RaiseTooManyValuesError(3);
+      if (unlikely(size != 2)) {
+        if (size > 2) __Pyx_RaiseTooManyValuesError(2);
         else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
-        {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       }
       #if CYTHON_COMPILING_IN_CPYTHON
       if (likely(PyTuple_CheckExact(sequence))) {
         __pyx_t_6 = PyTuple_GET_ITEM(sequence, 0); 
         __pyx_t_7 = PyTuple_GET_ITEM(sequence, 1); 
-        __pyx_t_8 = PyTuple_GET_ITEM(sequence, 2); 
       } else {
         __pyx_t_6 = PyList_GET_ITEM(sequence, 0); 
         __pyx_t_7 = PyList_GET_ITEM(sequence, 1); 
-        __pyx_t_8 = PyList_GET_ITEM(sequence, 2); 
       }
       __Pyx_INCREF(__pyx_t_6);
       __Pyx_INCREF(__pyx_t_7);
-      __Pyx_INCREF(__pyx_t_8);
       #else
-      __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_6 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_6);
-      __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_7 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_7);
-      __pyx_t_8 = PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-      __Pyx_GOTREF(__pyx_t_8);
       #endif
       __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
     } else {
       Py_ssize_t index = -1;
-      __pyx_t_9 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-      __Pyx_GOTREF(__pyx_t_9);
+      __pyx_t_8 = PyObject_GetIter(__pyx_t_5); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __Pyx_GOTREF(__pyx_t_8);
       __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
-      __pyx_t_10 = Py_TYPE(__pyx_t_9)->tp_iternext;
-      index = 0; __pyx_t_6 = __pyx_t_10(__pyx_t_9); if (unlikely(!__pyx_t_6)) goto __pyx_L6_unpacking_failed;
+      __pyx_t_9 = Py_TYPE(__pyx_t_8)->tp_iternext;
+      index = 0; __pyx_t_6 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_6)) goto __pyx_L6_unpacking_failed;
       __Pyx_GOTREF(__pyx_t_6);
-      index = 1; __pyx_t_7 = __pyx_t_10(__pyx_t_9); if (unlikely(!__pyx_t_7)) goto __pyx_L6_unpacking_failed;
+      index = 1; __pyx_t_7 = __pyx_t_9(__pyx_t_8); if (unlikely(!__pyx_t_7)) goto __pyx_L6_unpacking_failed;
       __Pyx_GOTREF(__pyx_t_7);
-      index = 2; __pyx_t_8 = __pyx_t_10(__pyx_t_9); if (unlikely(!__pyx_t_8)) goto __pyx_L6_unpacking_failed;
-      __Pyx_GOTREF(__pyx_t_8);
-      if (__Pyx_IternextUnpackEndCheck(__pyx_t_10(__pyx_t_9), 3) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-      __pyx_t_10 = NULL;
-      __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
+      if (__Pyx_IternextUnpackEndCheck(__pyx_t_9(__pyx_t_8), 2) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_9 = NULL;
+      __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
       goto __pyx_L7_unpacking_done;
       __pyx_L6_unpacking_failed:;
-      __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
-      __pyx_t_10 = NULL;
+      __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0;
+      __pyx_t_9 = NULL;
       if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);
-      {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __pyx_L7_unpacking_done:;
     }
     __Pyx_XDECREF_SET(__pyx_v_label, __pyx_t_6);
     __pyx_t_6 = 0;
-    __Pyx_XDECREF_SET(__pyx_v_cost, __pyx_t_7);
+    __Pyx_XDECREF_SET(__pyx_v_dist2next, __pyx_t_7);
     __pyx_t_7 = 0;
-    __Pyx_XDECREF_SET(__pyx_v_dist2next, __pyx_t_8);
-    __pyx_t_8 = 0;
 
-    /* "cdec/lattice.pxi":42
+    /* "cdec/lattice.pxi":44
  *         cdef lattice.LatticeArc* arc
- *         for (label, cost, dist2next) in arcs:
+ *         for (label, dist2next) in arcs:
  *             label_str = as_str(label)             # <<<<<<<<<<<<<<
- *             arc = new lattice.LatticeArc(TDConvert(label_str), cost, dist2next)
+ *             arc = new lattice.LatticeArc() #TDConvert(label_str), features, dist2next)
  *             self.lattice[0][index].push_back(arc[0])
  */
-    __pyx_t_5 = __pyx_f_4cdec_5_cdec_as_str(__pyx_v_label, NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 42; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_5 = __pyx_f_4cdec_5_cdec_as_str(__pyx_v_label, NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 44; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_5);
     __Pyx_XDECREF_SET(__pyx_v_label_str, ((PyObject*)__pyx_t_5));
     __pyx_t_5 = 0;
 
-    /* "cdec/lattice.pxi":43
- *         for (label, cost, dist2next) in arcs:
+    /* "cdec/lattice.pxi":45
+ *         for (label, dist2next) in arcs:
  *             label_str = as_str(label)
- *             arc = new lattice.LatticeArc(TDConvert(label_str), cost, dist2next)             # <<<<<<<<<<<<<<
+ *             arc = new lattice.LatticeArc() #TDConvert(label_str), features, dist2next)             # <<<<<<<<<<<<<<
  *             self.lattice[0][index].push_back(arc[0])
  *             del arc
  */
-    __pyx_t_11 = __Pyx_PyObject_AsString(__pyx_v_label_str); if (unlikely((!__pyx_t_11) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __pyx_t_12 = __pyx_PyFloat_AsDouble(__pyx_v_cost); if (unlikely((__pyx_t_12 == (double)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __pyx_t_13 = __Pyx_PyInt_As_int(__pyx_v_dist2next); if (unlikely((__pyx_t_13 == (int)-1) && PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 43; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-    __pyx_v_arc = new LatticeArc(TD::Convert(__pyx_t_11), __pyx_t_12, __pyx_t_13);
+    __pyx_v_arc = new LatticeArc();
 
-    /* "cdec/lattice.pxi":44
+    /* "cdec/lattice.pxi":46
  *             label_str = as_str(label)
- *             arc = new lattice.LatticeArc(TDConvert(label_str), cost, dist2next)
+ *             arc = new lattice.LatticeArc() #TDConvert(label_str), features, dist2next)
  *             self.lattice[0][index].push_back(arc[0])             # <<<<<<<<<<<<<<
  *             del arc
  * 
  */
-    ((__pyx_v_self->lattice[0])[__pyx_v_index]).push_back((__pyx_v_arc[0]));
+    try {
+      ((__pyx_v_self->lattice[0])[__pyx_v_index]).push_back((__pyx_v_arc[0]));
+    } catch(...) {
+      __Pyx_CppExn2PyErr();
+      {__pyx_filename = __pyx_f[4]; __pyx_lineno = 46; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    }
 
-    /* "cdec/lattice.pxi":45
- *             arc = new lattice.LatticeArc(TDConvert(label_str), cost, dist2next)
+    /* "cdec/lattice.pxi":47
+ *             arc = new lattice.LatticeArc() #TDConvert(label_str), features, dist2next)
  *             self.lattice[0][index].push_back(arc[0])
  *             del arc             # <<<<<<<<<<<<<<
  * 
@@ -16675,17 +16773,17 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_8__setitem__(struct __pyx_obj_4cdec_5_
  */
     delete __pyx_v_arc;
 
-    /* "cdec/lattice.pxi":41
+    /* "cdec/lattice.pxi":43
  *             raise IndexError('lattice index out of range')
  *         cdef lattice.LatticeArc* arc
- *         for (label, cost, dist2next) in arcs:             # <<<<<<<<<<<<<<
+ *         for (label, dist2next) in arcs:             # <<<<<<<<<<<<<<
  *             label_str = as_str(label)
- *             arc = new lattice.LatticeArc(TDConvert(label_str), cost, dist2next)
+ *             arc = new lattice.LatticeArc() #TDConvert(label_str), features, dist2next)
  */
   }
   __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
 
-  /* "cdec/lattice.pxi":37
+  /* "cdec/lattice.pxi":39
  *         return tuple(arcs)
  * 
  *     def __setitem__(self, int index, tuple arcs):             # <<<<<<<<<<<<<<
@@ -16702,19 +16800,17 @@ static int __pyx_pf_4cdec_5_cdec_7Lattice_8__setitem__(struct __pyx_obj_4cdec_5_
   __Pyx_XDECREF(__pyx_t_6);
   __Pyx_XDECREF(__pyx_t_7);
   __Pyx_XDECREF(__pyx_t_8);
-  __Pyx_XDECREF(__pyx_t_9);
   __Pyx_AddTraceback("cdec._cdec.Lattice.__setitem__", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __pyx_r = -1;
   __pyx_L0:;
   __Pyx_XDECREF(__pyx_v_label);
-  __Pyx_XDECREF(__pyx_v_cost);
   __Pyx_XDECREF(__pyx_v_dist2next);
   __Pyx_XDECREF(__pyx_v_label_str);
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
 }
 
-/* "cdec/lattice.pxi":47
+/* "cdec/lattice.pxi":49
  *             del arc
  * 
  *     def __len__(self):             # <<<<<<<<<<<<<<
@@ -16740,7 +16836,7 @@ static Py_ssize_t __pyx_pf_4cdec_5_cdec_7Lattice_10__len__(struct __pyx_obj_4cde
   __Pyx_RefNannyDeclarations
   __Pyx_RefNannySetupContext("__len__", 0);
 
-  /* "cdec/lattice.pxi":48
+  /* "cdec/lattice.pxi":50
  * 
  *     def __len__(self):
  *         return self.lattice.size()             # <<<<<<<<<<<<<<
@@ -16750,7 +16846,7 @@ static Py_ssize_t __pyx_pf_4cdec_5_cdec_7Lattice_10__len__(struct __pyx_obj_4cde
   __pyx_r = __pyx_v_self->lattice->size();
   goto __pyx_L0;
 
-  /* "cdec/lattice.pxi":47
+  /* "cdec/lattice.pxi":49
  *             del arc
  * 
  *     def __len__(self):             # <<<<<<<<<<<<<<
@@ -16764,7 +16860,7 @@ static Py_ssize_t __pyx_pf_4cdec_5_cdec_7Lattice_10__len__(struct __pyx_obj_4cde
   return __pyx_r;
 }
 
-/* "cdec/lattice.pxi":50
+/* "cdec/lattice.pxi":52
  *         return self.lattice.size()
  * 
  *     def __str__(self):             # <<<<<<<<<<<<<<
@@ -16795,7 +16891,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_12__str__(struct __pyx_obj_4cdec
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__str__", 0);
 
-  /* "cdec/lattice.pxi":51
+  /* "cdec/lattice.pxi":53
  * 
  *     def __str__(self):
  *         return str(hypergraph.AsPLF(self.lattice[0], True).c_str())             # <<<<<<<<<<<<<<
@@ -16803,21 +16899,21 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_12__str__(struct __pyx_obj_4cdec
  *     def __unicode__(self):
  */
   __Pyx_XDECREF(__pyx_r);
-  __pyx_t_1 = __Pyx_PyBytes_FromString(HypergraphIO::AsPLF((__pyx_v_self->lattice[0]), 1).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __Pyx_PyBytes_FromString(HypergraphIO::AsPLF((__pyx_v_self->lattice[0]), 1).c_str()); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_1);
   __Pyx_GIVEREF(__pyx_t_1);
   __pyx_t_1 = 0;
-  __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 51; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
   __pyx_r = __pyx_t_1;
   __pyx_t_1 = 0;
   goto __pyx_L0;
 
-  /* "cdec/lattice.pxi":50
+  /* "cdec/lattice.pxi":52
  *         return self.lattice.size()
  * 
  *     def __str__(self):             # <<<<<<<<<<<<<<
@@ -16837,7 +16933,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_12__str__(struct __pyx_obj_4cdec
   return __pyx_r;
 }
 
-/* "cdec/lattice.pxi":53
+/* "cdec/lattice.pxi":55
  *         return str(hypergraph.AsPLF(self.lattice[0], True).c_str())
  * 
  *     def __unicode__(self):             # <<<<<<<<<<<<<<
@@ -16868,7 +16964,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_14__unicode__(struct __pyx_obj_4
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("__unicode__", 0);
 
-  /* "cdec/lattice.pxi":54
+  /* "cdec/lattice.pxi":56
  * 
  *     def __unicode__(self):
  *         return unicode(str(self), 'utf8')             # <<<<<<<<<<<<<<
@@ -16876,15 +16972,15 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_14__unicode__(struct __pyx_obj_4
  *     def __iter__(self):
  */
   __Pyx_XDECREF(__pyx_r);
-  __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_INCREF(((PyObject *)__pyx_v_self));
   PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_self));
   __Pyx_GIVEREF(((PyObject *)__pyx_v_self));
-  __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-  __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_2);
   __Pyx_GIVEREF(__pyx_t_2);
@@ -16892,14 +16988,14 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_14__unicode__(struct __pyx_obj_4
   PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_n_s_utf8);
   __Pyx_GIVEREF(__pyx_n_s_utf8);
   __pyx_t_2 = 0;
-  __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyUnicode_Type))), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 54; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyUnicode_Type))), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
   __pyx_r = __pyx_t_2;
   __pyx_t_2 = 0;
   goto __pyx_L0;
 
-  /* "cdec/lattice.pxi":53
+  /* "cdec/lattice.pxi":55
  *         return str(hypergraph.AsPLF(self.lattice[0], True).c_str())
  * 
  *     def __unicode__(self):             # <<<<<<<<<<<<<<
@@ -16920,7 +17016,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_14__unicode__(struct __pyx_obj_4
 }
 static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_18generator18(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
 
-/* "cdec/lattice.pxi":56
+/* "cdec/lattice.pxi":58
  *         return unicode(str(self), 'utf8')
  * 
  *     def __iter__(self):             # <<<<<<<<<<<<<<
@@ -16959,7 +17055,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_16__iter__(struct __pyx_obj_4cde
   __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
   __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
   {
-    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_4cdec_5_cdec_7Lattice_18generator18, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Lattice___iter); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_4cdec_5_cdec_7Lattice_18generator18, (PyObject *) __pyx_cur_scope, __pyx_n_s_iter, __pyx_n_s_Lattice___iter); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_DECREF(__pyx_cur_scope);
     __Pyx_RefNannyFinishContext();
     return (PyObject *) gen;
@@ -16995,27 +17091,27 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_18generator18(__pyx_GeneratorObj
     return NULL;
   }
   __pyx_L3_first_run:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "cdec/lattice.pxi":58
+  /* "cdec/lattice.pxi":60
  *     def __iter__(self):
  *         cdef unsigned i
  *         for i in range(len(self)):             # <<<<<<<<<<<<<<
  *             yield self[i]
  * 
  */
-  __pyx_t_1 = PyObject_Length(((PyObject *)__pyx_cur_scope->__pyx_v_self)); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyObject_Length(((PyObject *)__pyx_cur_scope->__pyx_v_self)); if (unlikely(__pyx_t_1 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 60; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   for (__pyx_t_2 = 0; __pyx_t_2 < __pyx_t_1; __pyx_t_2+=1) {
     __pyx_cur_scope->__pyx_v_i = __pyx_t_2;
 
-    /* "cdec/lattice.pxi":59
+    /* "cdec/lattice.pxi":61
  *         cdef unsigned i
  *         for i in range(len(self)):
  *             yield self[i]             # <<<<<<<<<<<<<<
  * 
  *     def todot(self):
  */
-    __pyx_t_3 = __Pyx_GetItemInt(((PyObject *)__pyx_cur_scope->__pyx_v_self), __pyx_cur_scope->__pyx_v_i, unsigned int, 0, __Pyx_PyInt_From_unsigned_int, 0, 0, 1); if (unlikely(__pyx_t_3 == NULL)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;};
+    __pyx_t_3 = __Pyx_GetItemInt(((PyObject *)__pyx_cur_scope->__pyx_v_self), __pyx_cur_scope->__pyx_v_i, unsigned int, 0, __Pyx_PyInt_From_unsigned_int, 0, 0, 1); if (unlikely(__pyx_t_3 == NULL)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;};
     __Pyx_GOTREF(__pyx_t_3);
     __pyx_r = __pyx_t_3;
     __pyx_t_3 = 0;
@@ -17029,10 +17125,10 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_18generator18(__pyx_GeneratorObj
     __pyx_L6_resume_from_yield:;
     __pyx_t_1 = __pyx_cur_scope->__pyx_t_0;
     __pyx_t_2 = __pyx_cur_scope->__pyx_t_1;
-    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 59; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
 
-  /* "cdec/lattice.pxi":56
+  /* "cdec/lattice.pxi":58
  *         return unicode(str(self), 'utf8')
  * 
  *     def __iter__(self):             # <<<<<<<<<<<<<<
@@ -17054,7 +17150,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_18generator18(__pyx_GeneratorObj
   return NULL;
 }
 
-/* "cdec/lattice.pxi":61
+/* "cdec/lattice.pxi":63
  *             yield self[i]
  * 
  *     def todot(self):             # <<<<<<<<<<<<<<
@@ -17077,7 +17173,7 @@ static PyObject *__pyx_pw_4cdec_5_cdec_7Lattice_20todot(PyObject *__pyx_v_self,
 }
 static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_GeneratorObject *__pyx_generator, PyObject *__pyx_sent_value); /* proto */
 
-/* "cdec/lattice.pxi":63
+/* "cdec/lattice.pxi":65
  *     def todot(self):
  *         """lattice.todot() -> Representation of the lattice in GraphViz dot format."""
  *         def lines():             # <<<<<<<<<<<<<<
@@ -17117,7 +17213,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_5todot_lines(PyObject *__pyx_sel
   __Pyx_INCREF(((PyObject *)__pyx_cur_scope->__pyx_outer_scope));
   __Pyx_GIVEREF(__pyx_cur_scope->__pyx_outer_scope);
   {
-    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24, (PyObject *) __pyx_cur_scope, __pyx_n_s_lines, __pyx_n_s_todot_locals_lines); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_GeneratorObject *gen = __Pyx_Generator_New((__pyx_generator_body_t) __pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24, (PyObject *) __pyx_cur_scope, __pyx_n_s_lines, __pyx_n_s_todot_locals_lines); if (unlikely(!gen)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_DECREF(__pyx_cur_scope);
     __Pyx_RefNannyFinishContext();
     return (PyObject *) gen;
@@ -17167,9 +17263,9 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
     return NULL;
   }
   __pyx_L3_first_run:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "cdec/lattice.pxi":64
+  /* "cdec/lattice.pxi":66
  *         """lattice.todot() -> Representation of the lattice in GraphViz dot format."""
  *         def lines():
  *             yield 'digraph lattice {'             # <<<<<<<<<<<<<<
@@ -17184,9 +17280,9 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
   __pyx_generator->resume_label = 1;
   return __pyx_r;
   __pyx_L4_resume_from_yield:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 64; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "cdec/lattice.pxi":65
+  /* "cdec/lattice.pxi":67
  *         def lines():
  *             yield 'digraph lattice {'
  *             yield 'rankdir = LR;'             # <<<<<<<<<<<<<<
@@ -17201,9 +17297,9 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
   __pyx_generator->resume_label = 2;
   return __pyx_r;
   __pyx_L5_resume_from_yield:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "cdec/lattice.pxi":66
+  /* "cdec/lattice.pxi":68
  *             yield 'digraph lattice {'
  *             yield 'rankdir = LR;'
  *             yield 'node [shape=circle];'             # <<<<<<<<<<<<<<
@@ -17218,37 +17314,37 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
   __pyx_generator->resume_label = 3;
   return __pyx_r;
   __pyx_L6_resume_from_yield:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 66; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "cdec/lattice.pxi":67
+  /* "cdec/lattice.pxi":69
  *             yield 'rankdir = LR;'
  *             yield 'node [shape=circle];'
  *             for i in range(len(self)):             # <<<<<<<<<<<<<<
  *                 for label, weight, delta in self[i]:
  *                     yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"'))
  */
-  if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self)) { __Pyx_RaiseClosureNameError("self"); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;} }
+  if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self)) { __Pyx_RaiseClosureNameError("self"); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;} }
   __pyx_t_1 = ((PyObject *)__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self);
   __Pyx_INCREF(__pyx_t_1);
-  __pyx_t_2 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyObject_Length(__pyx_t_1); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-  __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_3);
   PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1);
   __Pyx_GIVEREF(__pyx_t_1);
   __pyx_t_1 = 0;
-  __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
   if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) {
     __pyx_t_3 = __pyx_t_1; __Pyx_INCREF(__pyx_t_3); __pyx_t_2 = 0;
     __pyx_t_4 = NULL;
   } else {
-    __pyx_t_2 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_2 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_3);
-    __pyx_t_4 = Py_TYPE(__pyx_t_3)->tp_iternext; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_4 = Py_TYPE(__pyx_t_3)->tp_iternext; if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
   for (;;) {
@@ -17256,16 +17352,16 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
       if (likely(PyList_CheckExact(__pyx_t_3))) {
         if (__pyx_t_2 >= PyList_GET_SIZE(__pyx_t_3)) break;
         #if CYTHON_COMPILING_IN_CPYTHON
-        __pyx_t_1 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_1 = PyList_GET_ITEM(__pyx_t_3, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         #else
-        __pyx_t_1 = PySequence_ITEM(__pyx_t_3, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_1 = PySequence_ITEM(__pyx_t_3, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         #endif
       } else {
         if (__pyx_t_2 >= PyTuple_GET_SIZE(__pyx_t_3)) break;
         #if CYTHON_COMPILING_IN_CPYTHON
-        __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_2); __Pyx_INCREF(__pyx_t_1); __pyx_t_2++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         #else
-        __pyx_t_1 = PySequence_ITEM(__pyx_t_3, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_1 = PySequence_ITEM(__pyx_t_3, __pyx_t_2); __pyx_t_2++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         #endif
       }
     } else {
@@ -17274,7 +17370,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
         PyObject* exc_type = PyErr_Occurred();
         if (exc_type) {
           if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
-          else {__pyx_filename = __pyx_f[4]; __pyx_lineno = 67; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          else {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         }
         break;
       }
@@ -17285,23 +17381,23 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
     __Pyx_GIVEREF(__pyx_t_1);
     __pyx_t_1 = 0;
 
-    /* "cdec/lattice.pxi":68
+    /* "cdec/lattice.pxi":70
  *             yield 'node [shape=circle];'
  *             for i in range(len(self)):
  *                 for label, weight, delta in self[i]:             # <<<<<<<<<<<<<<
  *                     yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"'))
  *             yield '%d [shape=doublecircle]' % len(self)
  */
-    if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self)) { __Pyx_RaiseClosureNameError("self"); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;} }
-    __pyx_t_1 = PyObject_GetItem(((PyObject *)__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self), __pyx_cur_scope->__pyx_v_i); if (unlikely(__pyx_t_1 == NULL)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;};
+    if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self)) { __Pyx_RaiseClosureNameError("self"); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} }
+    __pyx_t_1 = PyObject_GetItem(((PyObject *)__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self), __pyx_cur_scope->__pyx_v_i); if (unlikely(__pyx_t_1 == NULL)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;};
     __Pyx_GOTREF(__pyx_t_1);
     if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) {
       __pyx_t_5 = __pyx_t_1; __Pyx_INCREF(__pyx_t_5); __pyx_t_6 = 0;
       __pyx_t_7 = NULL;
     } else {
-      __pyx_t_6 = -1; __pyx_t_5 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_6 = -1; __pyx_t_5 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_5);
-      __pyx_t_7 = Py_TYPE(__pyx_t_5)->tp_iternext; if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_7 = Py_TYPE(__pyx_t_5)->tp_iternext; if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     }
     __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
     for (;;) {
@@ -17309,16 +17405,16 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
         if (likely(PyList_CheckExact(__pyx_t_5))) {
           if (__pyx_t_6 >= PyList_GET_SIZE(__pyx_t_5)) break;
           #if CYTHON_COMPILING_IN_CPYTHON
-          __pyx_t_1 = PyList_GET_ITEM(__pyx_t_5, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_1 = PyList_GET_ITEM(__pyx_t_5, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           #else
-          __pyx_t_1 = PySequence_ITEM(__pyx_t_5, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_1 = PySequence_ITEM(__pyx_t_5, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           #endif
         } else {
           if (__pyx_t_6 >= PyTuple_GET_SIZE(__pyx_t_5)) break;
           #if CYTHON_COMPILING_IN_CPYTHON
-          __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_5, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_5, __pyx_t_6); __Pyx_INCREF(__pyx_t_1); __pyx_t_6++; if (unlikely(0 < 0)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           #else
-          __pyx_t_1 = PySequence_ITEM(__pyx_t_5, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          __pyx_t_1 = PySequence_ITEM(__pyx_t_5, __pyx_t_6); __pyx_t_6++; if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           #endif
         }
       } else {
@@ -17327,7 +17423,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
           PyObject* exc_type = PyErr_Occurred();
           if (exc_type) {
             if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();
-            else {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+            else {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
           }
           break;
         }
@@ -17343,7 +17439,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
         if (unlikely(size != 3)) {
           if (size > 3) __Pyx_RaiseTooManyValuesError(3);
           else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);
-          {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+          {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         }
         #if CYTHON_COMPILING_IN_CPYTHON
         if (likely(PyTuple_CheckExact(sequence))) {
@@ -17359,17 +17455,17 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
         __Pyx_INCREF(__pyx_t_9);
         __Pyx_INCREF(__pyx_t_10);
         #else
-        __pyx_t_8 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_8 = PySequence_ITEM(sequence, 0); if (unlikely(!__pyx_t_8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_8);
-        __pyx_t_9 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_9 = PySequence_ITEM(sequence, 1); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_9);
-        __pyx_t_10 = PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_10 = PySequence_ITEM(sequence, 2); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_10);
         #endif
         __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
       } else {
         Py_ssize_t index = -1;
-        __pyx_t_11 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        __pyx_t_11 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_11)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __Pyx_GOTREF(__pyx_t_11);
         __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
         __pyx_t_12 = Py_TYPE(__pyx_t_11)->tp_iternext;
@@ -17379,7 +17475,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
         __Pyx_GOTREF(__pyx_t_9);
         index = 2; __pyx_t_10 = __pyx_t_12(__pyx_t_11); if (unlikely(!__pyx_t_10)) goto __pyx_L11_unpacking_failed;
         __Pyx_GOTREF(__pyx_t_10);
-        if (__Pyx_IternextUnpackEndCheck(__pyx_t_12(__pyx_t_11), 3) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        if (__Pyx_IternextUnpackEndCheck(__pyx_t_12(__pyx_t_11), 3) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __pyx_t_12 = NULL;
         __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
         goto __pyx_L12_unpacking_done;
@@ -17387,7 +17483,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
         __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0;
         __pyx_t_12 = NULL;
         if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);
-        {__pyx_filename = __pyx_f[4]; __pyx_lineno = 68; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+        {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
         __pyx_L12_unpacking_done:;
       }
       __Pyx_XGOTREF(__pyx_cur_scope->__pyx_v_label);
@@ -17403,21 +17499,21 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
       __Pyx_GIVEREF(__pyx_t_10);
       __pyx_t_10 = 0;
 
-      /* "cdec/lattice.pxi":69
+      /* "cdec/lattice.pxi":71
  *             for i in range(len(self)):
  *                 for label, weight, delta in self[i]:
  *                     yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"'))             # <<<<<<<<<<<<<<
  *             yield '%d [shape=doublecircle]' % len(self)
  *             yield '}'
  */
-      __pyx_t_1 = PyNumber_Add(__pyx_cur_scope->__pyx_v_i, __pyx_cur_scope->__pyx_v_delta); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_1 = PyNumber_Add(__pyx_cur_scope->__pyx_v_i, __pyx_cur_scope->__pyx_v_delta); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_1);
-      __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_label, __pyx_n_s_replace); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_10 = __Pyx_PyObject_GetAttrStr(__pyx_cur_scope->__pyx_v_label, __pyx_n_s_replace); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_10);
-      __pyx_t_9 = __Pyx_PyObject_Call(__pyx_t_10, __pyx_tuple__12, NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_9 = __Pyx_PyObject_Call(__pyx_t_10, __pyx_tuple__12, NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_9);
       __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
-      __pyx_t_10 = PyTuple_New(3); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_10 = PyTuple_New(3); if (unlikely(!__pyx_t_10)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_10);
       __Pyx_INCREF(__pyx_cur_scope->__pyx_v_i);
       PyTuple_SET_ITEM(__pyx_t_10, 0, __pyx_cur_scope->__pyx_v_i);
@@ -17428,7 +17524,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
       __Pyx_GIVEREF(__pyx_t_9);
       __pyx_t_1 = 0;
       __pyx_t_9 = 0;
-      __pyx_t_9 = __Pyx_PyString_Format(__pyx_kp_s_d_d_label_s, __pyx_t_10); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_9 = __Pyx_PyString_Format(__pyx_kp_s_d_d_label_s, __pyx_t_10); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_9);
       __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0;
       __pyx_r = __pyx_t_9;
@@ -17457,9 +17553,9 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
       __Pyx_XGOTREF(__pyx_t_5);
       __pyx_t_6 = __pyx_cur_scope->__pyx_t_4;
       __pyx_t_7 = __pyx_cur_scope->__pyx_t_5;
-      if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-      /* "cdec/lattice.pxi":68
+      /* "cdec/lattice.pxi":70
  *             yield 'node [shape=circle];'
  *             for i in range(len(self)):
  *                 for label, weight, delta in self[i]:             # <<<<<<<<<<<<<<
@@ -17469,7 +17565,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
     }
     __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;
 
-    /* "cdec/lattice.pxi":67
+    /* "cdec/lattice.pxi":69
  *             yield 'rankdir = LR;'
  *             yield 'node [shape=circle];'
  *             for i in range(len(self)):             # <<<<<<<<<<<<<<
@@ -17479,21 +17575,21 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
   }
   __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
 
-  /* "cdec/lattice.pxi":70
+  /* "cdec/lattice.pxi":72
  *                 for label, weight, delta in self[i]:
  *                     yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"'))
  *             yield '%d [shape=doublecircle]' % len(self)             # <<<<<<<<<<<<<<
  *             yield '}'
  *         return '\n'.join(lines()).encode('utf8')
  */
-  if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self)) { __Pyx_RaiseClosureNameError("self"); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;} }
+  if (unlikely(!__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self)) { __Pyx_RaiseClosureNameError("self"); {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;} }
   __pyx_t_3 = ((PyObject *)__pyx_cur_scope->__pyx_outer_scope->__pyx_v_self);
   __Pyx_INCREF(__pyx_t_3);
-  __pyx_t_2 = PyObject_Length(__pyx_t_3); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = PyObject_Length(__pyx_t_3); if (unlikely(__pyx_t_2 == -1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
-  __pyx_t_3 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = PyInt_FromSsize_t(__pyx_t_2); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_3);
-  __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_d_shape_doublecircle, __pyx_t_3); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_d_shape_doublecircle, __pyx_t_3); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_5);
   __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
   __pyx_r = __pyx_t_5;
@@ -17504,9 +17600,9 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
   __pyx_generator->resume_label = 5;
   return __pyx_r;
   __pyx_L14_resume_from_yield:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 70; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "cdec/lattice.pxi":71
+  /* "cdec/lattice.pxi":73
  *                     yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"'))
  *             yield '%d [shape=doublecircle]' % len(self)
  *             yield '}'             # <<<<<<<<<<<<<<
@@ -17521,9 +17617,9 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
   __pyx_generator->resume_label = 6;
   return __pyx_r;
   __pyx_L15_resume_from_yield:;
-  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (unlikely(!__pyx_sent_value)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 73; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "cdec/lattice.pxi":63
+  /* "cdec/lattice.pxi":65
  *     def todot(self):
  *         """lattice.todot() -> Representation of the lattice in GraphViz dot format."""
  *         def lines():             # <<<<<<<<<<<<<<
@@ -17551,7 +17647,7 @@ static PyObject *__pyx_gb_4cdec_5_cdec_7Lattice_5todot_2generator24(__pyx_Genera
   return NULL;
 }
 
-/* "cdec/lattice.pxi":61
+/* "cdec/lattice.pxi":63
  *             yield self[i]
  * 
  *     def todot(self):             # <<<<<<<<<<<<<<
@@ -17580,19 +17676,19 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_19todot(struct __pyx_obj_4cdec_5
   __Pyx_INCREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
   __Pyx_GIVEREF((PyObject *)__pyx_cur_scope->__pyx_v_self);
 
-  /* "cdec/lattice.pxi":63
+  /* "cdec/lattice.pxi":65
  *     def todot(self):
  *         """lattice.todot() -> Representation of the lattice in GraphViz dot format."""
  *         def lines():             # <<<<<<<<<<<<<<
  *             yield 'digraph lattice {'
  *             yield 'rankdir = LR;'
  */
-  __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4cdec_5_cdec_7Lattice_5todot_1lines, 0, __pyx_n_s_todot_locals_lines, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cdec__cdec, __pyx_d, ((PyObject *)__pyx_codeobj__15)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4cdec_5_cdec_7Lattice_5todot_1lines, 0, __pyx_n_s_todot_locals_lines, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cdec__cdec, __pyx_d, ((PyObject *)__pyx_codeobj__15)); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __pyx_v_lines = __pyx_t_1;
   __pyx_t_1 = 0;
 
-  /* "cdec/lattice.pxi":72
+  /* "cdec/lattice.pxi":74
  *             yield '%d [shape=doublecircle]' % len(self)
  *             yield '}'
  *         return '\n'.join(lines()).encode('utf8')             # <<<<<<<<<<<<<<
@@ -17600,22 +17696,22 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_19todot(struct __pyx_obj_4cdec_5
  *     def as_hypergraph(self):
  */
   __Pyx_XDECREF(__pyx_r);
-  __pyx_t_1 = __pyx_pf_4cdec_5_cdec_7Lattice_5todot_lines(__pyx_v_lines); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __pyx_pf_4cdec_5_cdec_7Lattice_5todot_lines(__pyx_v_lines); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  __pyx_t_2 = __Pyx_PyString_Join(__pyx_kp_s__16, __pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __Pyx_PyString_Join(__pyx_kp_s__16, __pyx_t_1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_encode); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
-  __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
   __pyx_r = __pyx_t_2;
   __pyx_t_2 = 0;
   goto __pyx_L0;
 
-  /* "cdec/lattice.pxi":61
+  /* "cdec/lattice.pxi":63
  *             yield self[i]
  * 
  *     def todot(self):             # <<<<<<<<<<<<<<
@@ -17637,7 +17733,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_19todot(struct __pyx_obj_4cdec_5
   return __pyx_r;
 }
 
-/* "cdec/lattice.pxi":74
+/* "cdec/lattice.pxi":76
  *         return '\n'.join(lines()).encode('utf8')
  * 
  *     def as_hypergraph(self):             # <<<<<<<<<<<<<<
@@ -17672,20 +17768,20 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_21as_hypergraph(struct __pyx_obj
   int __pyx_clineno = 0;
   __Pyx_RefNannySetupContext("as_hypergraph", 0);
 
-  /* "cdec/lattice.pxi":76
+  /* "cdec/lattice.pxi":78
  *     def as_hypergraph(self):
  *         """lattice.as_hypergraph() -> Hypergraph representation of the lattice."""
  *         cdef Hypergraph result = Hypergraph.__new__(Hypergraph)             # <<<<<<<<<<<<<<
  *         result.hg = new hypergraph.Hypergraph()
  *         cdef bytes plf = str(self)
  */
-  __pyx_t_1 = __pyx_tp_new_4cdec_5_cdec_Hypergraph(((PyTypeObject *)((PyObject*)__pyx_ptype_4cdec_5_cdec_Hypergraph)), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = __pyx_tp_new_4cdec_5_cdec_Hypergraph(((PyTypeObject *)((PyObject*)__pyx_ptype_4cdec_5_cdec_Hypergraph)), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
-  if (!(likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_4cdec_5_cdec_Hypergraph)))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 76; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (!(likely(__Pyx_TypeTest(__pyx_t_1, __pyx_ptype_4cdec_5_cdec_Hypergraph)))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_v_result = ((struct __pyx_obj_4cdec_5_cdec_Hypergraph *)__pyx_t_1);
   __pyx_t_1 = 0;
 
-  /* "cdec/lattice.pxi":77
+  /* "cdec/lattice.pxi":79
  *         """lattice.as_hypergraph() -> Hypergraph representation of the lattice."""
  *         cdef Hypergraph result = Hypergraph.__new__(Hypergraph)
  *         result.hg = new hypergraph.Hypergraph()             # <<<<<<<<<<<<<<
@@ -17694,35 +17790,35 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_21as_hypergraph(struct __pyx_obj
  */
   __pyx_v_result->hg = new Hypergraph();
 
-  /* "cdec/lattice.pxi":78
+  /* "cdec/lattice.pxi":80
  *         cdef Hypergraph result = Hypergraph.__new__(Hypergraph)
  *         result.hg = new hypergraph.Hypergraph()
  *         cdef bytes plf = str(self)             # <<<<<<<<<<<<<<
  *         hypergraph.ReadFromPLF(plf, result.hg)
  *         return result
  */
-  __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 80; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_1);
   __Pyx_INCREF(((PyObject *)__pyx_v_self));
   PyTuple_SET_ITEM(__pyx_t_1, 0, ((PyObject *)__pyx_v_self));
   __Pyx_GIVEREF(((PyObject *)__pyx_v_self));
-  __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)((PyObject*)(&PyString_Type))), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 80; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_2);
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-  if (!(likely(PyBytes_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_2)->tp_name), 0))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 78; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (!(likely(PyBytes_CheckExact(__pyx_t_2))||((__pyx_t_2) == Py_None)||(PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "bytes", Py_TYPE(__pyx_t_2)->tp_name), 0))) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 80; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_v_plf = ((PyObject*)__pyx_t_2);
   __pyx_t_2 = 0;
 
-  /* "cdec/lattice.pxi":79
+  /* "cdec/lattice.pxi":81
  *         result.hg = new hypergraph.Hypergraph()
  *         cdef bytes plf = str(self)
  *         hypergraph.ReadFromPLF(plf, result.hg)             # <<<<<<<<<<<<<<
  *         return result
  */
-  __pyx_t_3 = __pyx_convert_string_from_py_(__pyx_v_plf); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 79; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_v_plf); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 81; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   HypergraphIO::ReadFromPLF(__pyx_t_3, __pyx_v_result->hg);
 
-  /* "cdec/lattice.pxi":80
+  /* "cdec/lattice.pxi":82
  *         cdef bytes plf = str(self)
  *         hypergraph.ReadFromPLF(plf, result.hg)
  *         return result             # <<<<<<<<<<<<<<
@@ -17732,7 +17828,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Lattice_21as_hypergraph(struct __pyx_obj
   __pyx_r = ((PyObject *)__pyx_v_result);
   goto __pyx_L0;
 
-  /* "cdec/lattice.pxi":74
+  /* "cdec/lattice.pxi":76
  *         return '\n'.join(lines()).encode('utf8')
  * 
  *     def as_hypergraph(self):             # <<<<<<<<<<<<<<
@@ -17812,12 +17908,10 @@ static struct __pyx_obj_4cdec_5_cdec_SufficientStats *__pyx_f_4cdec_5_cdec_as_st
   __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 6; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
   if (__pyx_t_1) {
-    goto __pyx_L5_next_and;
   } else {
     __pyx_t_2 = __pyx_t_1;
     goto __pyx_L4_bool_binop_done;
   }
-  __pyx_L5_next_and:;
   __pyx_t_1 = __Pyx_TypeCheck(__pyx_v_y, ((PyObject*)__pyx_ptype_4cdec_5_cdec_SufficientStats)); 
   __pyx_t_4 = (__pyx_t_1 != 0);
   __pyx_t_2 = __pyx_t_4;
@@ -19587,7 +19681,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_16SegmentEvaluator_2evaluate(struct __pyx
   __pyx_t_3 = __pyx_f_4cdec_5_cdec_as_str(__pyx_t_1, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_t_3);
   __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
-  __pyx_t_5 = __pyx_convert_string_from_py_(__pyx_t_3); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_5 = __pyx_convert_string_from_py_std__in_string(__pyx_t_3); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 116; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
   TD::ConvertSentence(__pyx_t_5, (&__pyx_v_hyp));
 
@@ -20109,7 +20203,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_6Scorer_4__call__(struct __pyx_obj_4cdec_
     __pyx_t_9 = __pyx_f_4cdec_5_cdec_as_str(__pyx_t_7, NULL); if (unlikely(!__pyx_t_9)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_GOTREF(__pyx_t_9);
     __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0;
-    __pyx_t_11 = __pyx_convert_string_from_py_(__pyx_t_9); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    __pyx_t_11 = __pyx_convert_string_from_py_std__in_string(__pyx_t_9); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 143; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0;
     TD::ConvertSentence(__pyx_t_11, __pyx_v_refv);
 
@@ -20120,7 +20214,12 @@ static PyObject *__pyx_pf_4cdec_5_cdec_6Scorer_4__call__(struct __pyx_obj_4cdec_
  *             del refv
  *         cdef unsigned i
  */
-    __pyx_v_refsv->push_back((__pyx_v_refv[0]));
+    try {
+      __pyx_v_refsv->push_back((__pyx_v_refv[0]));
+    } catch(...) {
+      __Pyx_CppExn2PyErr();
+      {__pyx_filename = __pyx_f[5]; __pyx_lineno = 144; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    }
 
     /* "cdec/mteval.pxi":145
  *             ConvertSentence(as_str(ref.strip()), refv)
@@ -20575,7 +20674,12 @@ static void __pyx_f_4cdec_5_cdec__compute_sufficient_stats(void *__pyx_v_metric_
     {__pyx_filename = __pyx_f[5]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   }
   __pyx_t_8 = PyList_GET_SIZE(__pyx_v_ss); if (unlikely(__pyx_t_8 == -1)) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_v_out->fields.resize(__pyx_t_8);
+  try {
+    __pyx_v_out->fields.resize(__pyx_t_8);
+  } catch(...) {
+    __Pyx_CppExn2PyErr();
+    {__pyx_filename = __pyx_f[5]; __pyx_lineno = 175; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  }
 
   /* "cdec/mteval.pxi":176
  *     cdef list ss = metric.evaluate(str(hyp.c_str()), refs_)
@@ -22042,60 +22146,46 @@ static int __pyx_pf_4cdec_5_cdec_7Decoder___init__(struct __pyx_obj_4cdec_5_cdec
     __pyx_t_3 = __pyx_v_formalism;
     __pyx_t_1 = (__Pyx_PyString_Equals(__pyx_t_3, __pyx_n_s_scfg, Py_NE)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     if (__pyx_t_1) {
-      goto __pyx_L12_next_and;
     } else {
       __pyx_t_2 = __pyx_t_1;
       goto __pyx_L5_bool_binop_done;
     }
-    __pyx_L12_next_and:;
     __pyx_t_1 = (__Pyx_PyString_Equals(__pyx_t_3, __pyx_n_s_fst, Py_NE)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     if (__pyx_t_1) {
-      goto __pyx_L11_next_and;
     } else {
       __pyx_t_2 = __pyx_t_1;
       goto __pyx_L5_bool_binop_done;
     }
-    __pyx_L11_next_and:;
     __pyx_t_1 = (__Pyx_PyString_Equals(__pyx_t_3, __pyx_n_s_lextrans, Py_NE)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     if (__pyx_t_1) {
-      goto __pyx_L10_next_and;
     } else {
       __pyx_t_2 = __pyx_t_1;
       goto __pyx_L5_bool_binop_done;
     }
-    __pyx_L10_next_and:;
     __pyx_t_1 = (__Pyx_PyString_Equals(__pyx_t_3, __pyx_n_s_pb, Py_NE)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     if (__pyx_t_1) {
-      goto __pyx_L9_next_and;
     } else {
       __pyx_t_2 = __pyx_t_1;
       goto __pyx_L5_bool_binop_done;
     }
-    __pyx_L9_next_and:;
     __pyx_t_1 = (__Pyx_PyString_Equals(__pyx_t_3, __pyx_n_s_csplit, Py_NE)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     if (__pyx_t_1) {
-      goto __pyx_L8_next_and;
     } else {
       __pyx_t_2 = __pyx_t_1;
       goto __pyx_L5_bool_binop_done;
     }
-    __pyx_L8_next_and:;
     __pyx_t_1 = (__Pyx_PyString_Equals(__pyx_t_3, __pyx_n_s_tagger, Py_NE)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     if (__pyx_t_1) {
-      goto __pyx_L7_next_and;
     } else {
       __pyx_t_2 = __pyx_t_1;
       goto __pyx_L5_bool_binop_done;
     }
-    __pyx_L7_next_and:;
     __pyx_t_1 = (__Pyx_PyString_Equals(__pyx_t_3, __pyx_n_s_lexalign, Py_NE)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     if (__pyx_t_1) {
-      goto __pyx_L6_next_and;
     } else {
       __pyx_t_2 = __pyx_t_1;
       goto __pyx_L5_bool_binop_done;
     }
-    __pyx_L6_next_and:;
     __pyx_t_1 = (__Pyx_PyString_Equals(__pyx_t_3, __pyx_n_s_t2s, Py_NE)); if (unlikely(__pyx_t_1 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 53; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     __pyx_t_2 = __pyx_t_1;
     __pyx_L5_bool_binop_done:;
@@ -23429,7 +23519,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Decoder_6translate(struct __pyx_obj_4cde
  */
       __pyx_t_3 = __pyx_f_4cdec_5_cdec_as_str(__pyx_v_grammar, NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_GOTREF(__pyx_t_3);
-      __pyx_t_6 = __pyx_convert_string_from_py_(__pyx_t_3); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+      __pyx_t_6 = __pyx_convert_string_from_py_std__in_string(__pyx_t_3); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 108; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
       __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
       __pyx_v_self->dec->AddSupplementalGrammarFromString(__pyx_t_6);
       goto __pyx_L5;
@@ -23475,7 +23565,7 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Decoder_6translate(struct __pyx_obj_4cde
  *         if observer.hypergraph == NULL:
  *             raise ParseFailed()
  */
-  __pyx_t_6 = __pyx_convert_string_from_py_(__pyx_v_input_str); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_t_6 = __pyx_convert_string_from_py_std__in_string(__pyx_v_input_str); if (unlikely(PyErr_Occurred())) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 112; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_v_self->dec->Decode(__pyx_t_6, (&__pyx_v_observer));
 
   /* "cdec/_cdec.pyx":113
@@ -23575,13 +23665,13 @@ static PyObject *__pyx_pf_4cdec_5_cdec_7Decoder_6translate(struct __pyx_obj_4cde
 
 /* "string.from_py":13
  * 
- * @cname("__pyx_convert_string_from_py_")
- * cdef string __pyx_convert_string_from_py_(object o) except *:             # <<<<<<<<<<<<<<
+ * @cname("__pyx_convert_string_from_py_std__in_string")
+ * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *:             # <<<<<<<<<<<<<<
  *     cdef Py_ssize_t length
  *     cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length)
  */
 
-static std::string __pyx_convert_string_from_py_(PyObject *__pyx_v_o) {
+static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) {
   Py_ssize_t __pyx_v_length;
   char *__pyx_v_data;
   std::string __pyx_r;
@@ -23590,10 +23680,10 @@ static std::string __pyx_convert_string_from_py_(PyObject *__pyx_v_o) {
   int __pyx_lineno = 0;
   const char *__pyx_filename = NULL;
   int __pyx_clineno = 0;
-  __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_", 0);
+  __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0);
 
   /* "string.from_py":15
- * cdef string __pyx_convert_string_from_py_(object o) except *:
+ * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *:
  *     cdef Py_ssize_t length
  *     cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length)             # <<<<<<<<<<<<<<
  *     return string(data, length)
@@ -23614,15 +23704,15 @@ static std::string __pyx_convert_string_from_py_(PyObject *__pyx_v_o) {
 
   /* "string.from_py":13
  * 
- * @cname("__pyx_convert_string_from_py_")
- * cdef string __pyx_convert_string_from_py_(object o) except *:             # <<<<<<<<<<<<<<
+ * @cname("__pyx_convert_string_from_py_std__in_string")
+ * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *:             # <<<<<<<<<<<<<<
  *     cdef Py_ssize_t length
  *     cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length)
  */
 
   /* function exit code */
   __pyx_L1_error:;
-  __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_", __pyx_clineno, __pyx_lineno, __pyx_filename);
+  __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename);
   __pyx_L0:;
   __Pyx_RefNannyFinishContext();
   return __pyx_r;
@@ -29343,9 +29433,6 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_n_s_TER, __pyx_k_TER, sizeof(__pyx_k_TER), 0, 0, 1, 1},
   {&__pyx_n_s_TRule___get, __pyx_k_TRule___get, sizeof(__pyx_k_TRule___get), 0, 0, 1, 1},
   {&__pyx_n_s_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 0, 0, 1, 1},
-  {&__pyx_kp_s_Users_waziz_workspace_mtm14_my, __pyx_k_Users_waziz_workspace_mtm14_my, sizeof(__pyx_k_Users_waziz_workspace_mtm14_my), 0, 0, 1, 0},
-  {&__pyx_kp_s_Users_waziz_workspace_mtm14_my_2, __pyx_k_Users_waziz_workspace_mtm14_my_2, sizeof(__pyx_k_Users_waziz_workspace_mtm14_my_2), 0, 0, 1, 0},
-  {&__pyx_kp_s_Users_waziz_workspace_mtm14_my_3, __pyx_k_Users_waziz_workspace_mtm14_my_3, sizeof(__pyx_k_Users_waziz_workspace_mtm14_my_3), 0, 0, 1, 0},
   {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1},
   {&__pyx_kp_s__10, __pyx_k__10, sizeof(__pyx_k__10), 0, 0, 1, 0},
   {&__pyx_kp_s__11, __pyx_k__11, sizeof(__pyx_k__11), 0, 0, 1, 0},
@@ -29403,6 +29490,9 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = {
   {&__pyx_n_s_get, __pyx_k_get, sizeof(__pyx_k_get), 0, 0, 1, 1},
   {&__pyx_n_s_get_2, __pyx_k_get_2, sizeof(__pyx_k_get_2), 0, 0, 1, 1},
   {&__pyx_n_s_grammar, __pyx_k_grammar, sizeof(__pyx_k_grammar), 0, 0, 1, 1},
+  {&__pyx_kp_s_home_pks_src_cdec_dtrain_python, __pyx_k_home_pks_src_cdec_dtrain_python, sizeof(__pyx_k_home_pks_src_cdec_dtrain_python), 0, 0, 1, 0},
+  {&__pyx_kp_s_home_pks_src_cdec_dtrain_python_2, __pyx_k_home_pks_src_cdec_dtrain_python_2, sizeof(__pyx_k_home_pks_src_cdec_dtrain_python_2), 0, 0, 1, 0},
+  {&__pyx_kp_s_home_pks_src_cdec_dtrain_python_3, __pyx_k_home_pks_src_cdec_dtrain_python_3, sizeof(__pyx_k_home_pks_src_cdec_dtrain_python_3), 0, 0, 1, 0},
   {&__pyx_n_s_hyp, __pyx_k_hyp, sizeof(__pyx_k_hyp), 0, 0, 1, 1},
   {&__pyx_n_s_hypergraph, __pyx_k_hypergraph, sizeof(__pyx_k_hypergraph), 0, 0, 1, 1},
   {&__pyx_n_s_i, __pyx_k_i, sizeof(__pyx_k_i), 0, 0, 1, 1},
@@ -29500,8 +29590,8 @@ static int __Pyx_InitCachedBuiltins(void) {
   __pyx_builtin_super = __Pyx_GetBuiltinName(__pyx_n_s_super); if (!__pyx_builtin_super) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 232; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_eval = __Pyx_GetBuiltinName(__pyx_n_s_eval); if (!__pyx_builtin_eval) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 208; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_n_s_enumerate); if (!__pyx_builtin_enumerate) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 14; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
-  __pyx_builtin_IndexError = __Pyx_GetBuiltinName(__pyx_n_s_IndexError); if (!__pyx_builtin_IndexError) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_n_s_enumerate); if (!__pyx_builtin_enumerate) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 15; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_builtin_IndexError = __Pyx_GetBuiltinName(__pyx_n_s_IndexError); if (!__pyx_builtin_IndexError) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_builtin_open = __Pyx_GetBuiltinName(__pyx_n_s_open); if (!__pyx_builtin_open) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 90; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   return 0;
   __pyx_L1_error:;
@@ -29575,59 +29665,59 @@ static int __Pyx_InitCachedConstants(void) {
   __Pyx_GOTREF(__pyx_tuple__7);
   __Pyx_GIVEREF(__pyx_tuple__7);
 
-  /* "cdec/lattice.pxi":26
+  /* "cdec/lattice.pxi":27
  *     def __getitem__(self, int index):
  *         if not 0 <= index < len(self):
  *             raise IndexError('lattice index out of range')             # <<<<<<<<<<<<<<
  *         arcs = []
  *         cdef vector[lattice.LatticeArc] arc_vector = self.lattice[0][index]
  */
-  __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_s_lattice_index_out_of_range); if (unlikely(!__pyx_tuple__8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 26; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_tuple__8 = PyTuple_Pack(1, __pyx_kp_s_lattice_index_out_of_range); if (unlikely(!__pyx_tuple__8)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 27; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_tuple__8);
   __Pyx_GIVEREF(__pyx_tuple__8);
 
-  /* "cdec/lattice.pxi":39
+  /* "cdec/lattice.pxi":41
  *     def __setitem__(self, int index, tuple arcs):
  *         if not 0 <= index < len(self):
  *             raise IndexError('lattice index out of range')             # <<<<<<<<<<<<<<
  *         cdef lattice.LatticeArc* arc
- *         for (label, cost, dist2next) in arcs:
+ *         for (label, dist2next) in arcs:
  */
-  __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_lattice_index_out_of_range); if (unlikely(!__pyx_tuple__9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 39; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_tuple__9 = PyTuple_Pack(1, __pyx_kp_s_lattice_index_out_of_range); if (unlikely(!__pyx_tuple__9)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 41; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_tuple__9);
   __Pyx_GIVEREF(__pyx_tuple__9);
 
-  /* "cdec/lattice.pxi":69
+  /* "cdec/lattice.pxi":71
  *             for i in range(len(self)):
  *                 for label, weight, delta in self[i]:
  *                     yield '%d -> %d [label="%s"];' % (i, i+delta, label.replace('"', '\\"'))             # <<<<<<<<<<<<<<
  *             yield '%d [shape=doublecircle]' % len(self)
  *             yield '}'
  */
-  __pyx_tuple__12 = PyTuple_Pack(2, __pyx_kp_s__10, __pyx_kp_s__11); if (unlikely(!__pyx_tuple__12)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 69; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_tuple__12 = PyTuple_Pack(2, __pyx_kp_s__10, __pyx_kp_s__11); if (unlikely(!__pyx_tuple__12)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 71; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_tuple__12);
   __Pyx_GIVEREF(__pyx_tuple__12);
 
-  /* "cdec/lattice.pxi":63
+  /* "cdec/lattice.pxi":65
  *     def todot(self):
  *         """lattice.todot() -> Representation of the lattice in GraphViz dot format."""
  *         def lines():             # <<<<<<<<<<<<<<
  *             yield 'digraph lattice {'
  *             yield 'rankdir = LR;'
  */
-  __pyx_tuple__14 = PyTuple_Pack(4, __pyx_n_s_i, __pyx_n_s_label, __pyx_n_s_weight, __pyx_n_s_delta); if (unlikely(!__pyx_tuple__14)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_tuple__14 = PyTuple_Pack(4, __pyx_n_s_i, __pyx_n_s_label, __pyx_n_s_weight, __pyx_n_s_delta); if (unlikely(!__pyx_tuple__14)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_tuple__14);
   __Pyx_GIVEREF(__pyx_tuple__14);
-  __pyx_codeobj__15 = (PyObject*)__Pyx_PyCode_New(0, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_waziz_workspace_mtm14_my, __pyx_n_s_lines, 63, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__15)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_codeobj__15 = (PyObject*)__Pyx_PyCode_New(0, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_pks_src_cdec_dtrain_python, __pyx_n_s_lines, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__15)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
-  /* "cdec/lattice.pxi":72
+  /* "cdec/lattice.pxi":74
  *             yield '%d [shape=doublecircle]' % len(self)
  *             yield '}'
  *         return '\n'.join(lines()).encode('utf8')             # <<<<<<<<<<<<<<
  * 
  *     def as_hypergraph(self):
  */
-  __pyx_tuple__17 = PyTuple_Pack(1, __pyx_n_s_utf8); if (unlikely(!__pyx_tuple__17)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 72; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_tuple__17 = PyTuple_Pack(1, __pyx_n_s_utf8); if (unlikely(!__pyx_tuple__17)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 74; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_tuple__17);
   __Pyx_GIVEREF(__pyx_tuple__17);
 
@@ -29685,7 +29775,7 @@ static int __Pyx_InitCachedConstants(void) {
   __pyx_tuple__24 = PyTuple_Pack(3, __pyx_n_s_phrase_2, __pyx_n_s_genexpr, __pyx_n_s_genexpr); if (unlikely(!__pyx_tuple__24)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 5; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_tuple__24);
   __Pyx_GIVEREF(__pyx_tuple__24);
-  __pyx_codeobj__25 = (PyObject*)__Pyx_PyCode_New(1, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_waziz_workspace_mtm14_my_2, __pyx_n_s_phrase, 5, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__25)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 5; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_codeobj__25 = (PyObject*)__Pyx_PyCode_New(1, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_pks_src_cdec_dtrain_python_2, __pyx_n_s_phrase, 5, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__25)) {__pyx_filename = __pyx_f[2]; __pyx_lineno = 5; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
   /* "cdec/mteval.pxi":197
  *         return []
@@ -29749,7 +29839,7 @@ static int __Pyx_InitCachedConstants(void) {
   __pyx_tuple__31 = PyTuple_Pack(1, __pyx_n_s_yn); if (unlikely(!__pyx_tuple__31)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_tuple__31);
   __Pyx_GIVEREF(__pyx_tuple__31);
-  __pyx_codeobj__32 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__31, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_waziz_workspace_mtm14_my_3, __pyx_n_s_set_silent, 28, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__32)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_codeobj__32 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__31, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_pks_src_cdec_dtrain_python_3, __pyx_n_s_set_silent, 28, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__32)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 28; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
 
   /* "cdec/_cdec.pyx":32
  *     SetSilent(yn)
@@ -29761,7 +29851,7 @@ static int __Pyx_InitCachedConstants(void) {
   __pyx_tuple__33 = PyTuple_Pack(5, __pyx_n_s_config, __pyx_n_s_key, __pyx_n_s_value, __pyx_n_s_name, __pyx_n_s_info); if (unlikely(!__pyx_tuple__33)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_GOTREF(__pyx_tuple__33);
   __Pyx_GIVEREF(__pyx_tuple__33);
-  __pyx_codeobj__34 = (PyObject*)__Pyx_PyCode_New(1, 0, 5, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__33, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_Users_waziz_workspace_mtm14_my_3, __pyx_n_s_make_config, 32, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__34)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  __pyx_codeobj__34 = (PyObject*)__Pyx_PyCode_New(1, 0, 5, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__33, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_pks_src_cdec_dtrain_python_3, __pyx_n_s_make_config, 32, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__34)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 32; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __Pyx_RefNannyFinishContext();
   return 0;
   __pyx_L1_error:;
@@ -29988,11 +30078,11 @@ PyMODINIT_FUNC PyInit__cdec(void)
   if (__Pyx_SetVtable(__pyx_type_4cdec_5_cdec_HypergraphNode.tp_dict, __pyx_vtabptr_4cdec_5_cdec_HypergraphNode) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 309; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   if (PyObject_SetAttrString(__pyx_m, "HypergraphNode", (PyObject *)&__pyx_type_4cdec_5_cdec_HypergraphNode) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 309; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_4cdec_5_cdec_HypergraphNode = &__pyx_type_4cdec_5_cdec_HypergraphNode;
-  if (PyType_Ready(&__pyx_type_4cdec_5_cdec_Lattice) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyType_Ready(&__pyx_type_4cdec_5_cdec_Lattice) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 4; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_type_4cdec_5_cdec_Lattice.tp_print = 0;
   #if CYTHON_COMPILING_IN_CPYTHON
   {
-    PyObject *wrapper = PyObject_GetAttrString((PyObject *)&__pyx_type_4cdec_5_cdec_Lattice, "__init__"); if (unlikely(!wrapper)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+    PyObject *wrapper = PyObject_GetAttrString((PyObject *)&__pyx_type_4cdec_5_cdec_Lattice, "__init__"); if (unlikely(!wrapper)) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 4; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
     if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {
       __pyx_wrapperbase_4cdec_5_cdec_7Lattice_2__init__ = *((PyWrapperDescrObject *)wrapper)->d_base;
       __pyx_wrapperbase_4cdec_5_cdec_7Lattice_2__init__.doc = __pyx_doc_4cdec_5_cdec_7Lattice_2__init__;
@@ -30000,7 +30090,7 @@ PyMODINIT_FUNC PyInit__cdec(void)
     }
   }
   #endif
-  if (PyObject_SetAttrString(__pyx_m, "Lattice", (PyObject *)&__pyx_type_4cdec_5_cdec_Lattice) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 3; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyObject_SetAttrString(__pyx_m, "Lattice", (PyObject *)&__pyx_type_4cdec_5_cdec_Lattice) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 4; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_ptype_4cdec_5_cdec_Lattice = &__pyx_type_4cdec_5_cdec_Lattice;
   if (PyType_Ready(&__pyx_type_4cdec_5_cdec_Candidate) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 12; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_type_4cdec_5_cdec_Candidate.tp_print = 0;
@@ -30106,13 +30196,13 @@ PyMODINIT_FUNC PyInit__cdec(void)
   if (PyType_Ready(&__pyx_type_4cdec_5_cdec___pyx_scope_struct_21___get__) < 0) {__pyx_filename = __pyx_f[3]; __pyx_lineno = 329; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_type_4cdec_5_cdec___pyx_scope_struct_21___get__.tp_print = 0;
   __pyx_ptype_4cdec_5_cdec___pyx_scope_struct_21___get__ = &__pyx_type_4cdec_5_cdec___pyx_scope_struct_21___get__;
-  if (PyType_Ready(&__pyx_type_4cdec_5_cdec___pyx_scope_struct_22___iter__) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 56; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyType_Ready(&__pyx_type_4cdec_5_cdec___pyx_scope_struct_22___iter__) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 58; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_type_4cdec_5_cdec___pyx_scope_struct_22___iter__.tp_print = 0;
   __pyx_ptype_4cdec_5_cdec___pyx_scope_struct_22___iter__ = &__pyx_type_4cdec_5_cdec___pyx_scope_struct_22___iter__;
-  if (PyType_Ready(&__pyx_type_4cdec_5_cdec___pyx_scope_struct_23_todot) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 61; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyType_Ready(&__pyx_type_4cdec_5_cdec___pyx_scope_struct_23_todot) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_type_4cdec_5_cdec___pyx_scope_struct_23_todot.tp_print = 0;
   __pyx_ptype_4cdec_5_cdec___pyx_scope_struct_23_todot = &__pyx_type_4cdec_5_cdec___pyx_scope_struct_23_todot;
-  if (PyType_Ready(&__pyx_type_4cdec_5_cdec___pyx_scope_struct_24_lines) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 63; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+  if (PyType_Ready(&__pyx_type_4cdec_5_cdec___pyx_scope_struct_24_lines) < 0) {__pyx_filename = __pyx_f[4]; __pyx_lineno = 65; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
   __pyx_type_4cdec_5_cdec___pyx_scope_struct_24_lines.tp_print = 0;
   __pyx_ptype_4cdec_5_cdec___pyx_scope_struct_24_lines = &__pyx_type_4cdec_5_cdec___pyx_scope_struct_24_lines;
   if (PyType_Ready(&__pyx_type_4cdec_5_cdec___pyx_scope_struct_25___iter__) < 0) {__pyx_filename = __pyx_f[5]; __pyx_lineno = 47; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
@@ -30338,8 +30428,8 @@ PyMODINIT_FUNC PyInit__cdec(void)
 
   /* "string.from_py":13
  * 
- * @cname("__pyx_convert_string_from_py_")
- * cdef string __pyx_convert_string_from_py_(object o) except *:             # <<<<<<<<<<<<<<
+ * @cname("__pyx_convert_string_from_py_std__in_string")
+ * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *:             # <<<<<<<<<<<<<<
  *     cdef Py_ssize_t length
  *     cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length)
  */
@@ -30356,7 +30446,6 @@ PyMODINIT_FUNC PyInit__cdec(void)
   if (__pyx_m) {
     if (__pyx_d) {
       __Pyx_AddTraceback("init cdec._cdec", __pyx_clineno, __pyx_lineno, __pyx_filename);
-      Py_DECREF(__pyx_d); __pyx_d = 0;
     }
     Py_DECREF(__pyx_m); __pyx_m = 0;
   } else if (!PyErr_Occurred()) {
@@ -30371,7 +30460,7 @@ PyMODINIT_FUNC PyInit__cdec(void)
   #endif
 }
 
-/* Runtime support code */
+/* --- Runtime support code --- */
 #if CYTHON_REFNANNY
 static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) {
     PyObject *m = NULL, *p = NULL;
@@ -30635,6 +30724,13 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject
     }
     PyErr_SetObject(type, value);
     if (tb) {
+#if CYTHON_COMPILING_IN_PYPY
+        PyObject *tmp_type, *tmp_value, *tmp_tb;
+        PyErr_Fetch(tmp_type, tmp_value, tmp_tb);
+        Py_INCREF(tb);
+        PyErr_Restore(tmp_type, tmp_value, tb);
+        Py_XDECREF(tmp_tb);
+#else
         PyThreadState *tstate = PyThreadState_GET();
         PyObject* tmp_tb = tstate->curexc_traceback;
         if (tb != tmp_tb) {
@@ -30642,6 +30738,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject
             tstate->curexc_traceback = tb;
             Py_XDECREF(tmp_tb);
         }
+#endif
     }
 bad:
     Py_XDECREF(owned_instance);
@@ -31564,9 +31661,6 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = {
     {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0},
     {0, 0, 0, 0, 0}
 };
-#ifndef PY_WRITE_RESTRICTED
-#define PY_WRITE_RESTRICTED WRITE_RESTRICTED
-#endif
 static PyMemberDef __pyx_CyFunction_members[] = {
     {(char *) "__module__", T_OBJECT, offsetof(__pyx_CyFunctionObject, func.m_module), PY_WRITE_RESTRICTED, 0},
     {0, 0, 0,  0, 0}
@@ -31685,12 +31779,11 @@ static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObj
     if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) {
         if (type == NULL)
             type = (PyObject *)(Py_TYPE(obj));
-        return PyMethod_New(func,
-                            type, (PyObject *)(Py_TYPE(type)));
+        return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type)));
     }
     if (obj == Py_None)
         obj = NULL;
-    return PyMethod_New(func, obj, type);
+    return __Pyx_PyMethod_New(func, obj, type);
 }
 static PyObject*
 __Pyx_CyFunction_repr(__pyx_CyFunctionObject *op)
@@ -31722,7 +31815,7 @@ static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject
             if (size == 0)
                 return (*meth)(self, NULL);
             PyErr_Format(PyExc_TypeError,
-                "%.200s() takes no arguments (%zd given)",
+                "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)",
                 f->m_ml->ml_name, size);
             return NULL;
         }
@@ -31733,7 +31826,7 @@ static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject
             if (size == 1)
                 return (*meth)(self, PyTuple_GET_ITEM(arg, 0));
             PyErr_Format(PyExc_TypeError,
-                "%.200s() takes exactly one argument (%zd given)",
+                "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)",
                 f->m_ml->ml_name, size);
             return NULL;
         }
@@ -32333,32 +32426,6 @@ bad:
     Py_XDECREF(py_frame);
 }
 
-static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) {
-    const unsigned int neg_one = (unsigned int) -1, const_zero = 0;
-    const int is_unsigned = neg_one > const_zero;
-    if (is_unsigned) {
-        if (sizeof(unsigned int) < sizeof(long)) {
-            return PyInt_FromLong((long) value);
-        } else if (sizeof(unsigned int) <= sizeof(unsigned long)) {
-            return PyLong_FromUnsignedLong((unsigned long) value);
-        } else if (sizeof(unsigned int) <= sizeof(unsigned long long)) {
-            return PyLong_FromUnsignedLongLong((unsigned long long) value);
-        }
-    } else {
-        if (sizeof(unsigned int) <= sizeof(long)) {
-            return PyInt_FromLong((long) value);
-        } else if (sizeof(unsigned int) <= sizeof(long long)) {
-            return PyLong_FromLongLong((long long) value);
-        }
-    }
-    {
-        int one = 1; int little = (int)*(unsigned char *)&one;
-        unsigned char *bytes = (unsigned char *)&value;
-        return _PyLong_FromByteArray(bytes, sizeof(unsigned int),
-                                     little, !is_unsigned);
-    }
-}
-
 #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)       \
     {                                                                     \
         func_type value = func_value;                                     \
@@ -32380,101 +32447,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value)
  #endif
 #endif
 
-static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) {
-    const unsigned int neg_one = (unsigned int) -1, const_zero = 0;
-    const int is_unsigned = neg_one > const_zero;
-#if PY_MAJOR_VERSION < 3
-    if (likely(PyInt_Check(x))) {
-        if (sizeof(unsigned int) < sizeof(long)) {
-            __PYX_VERIFY_RETURN_INT(unsigned int, long, PyInt_AS_LONG(x))
-        } else {
-            long val = PyInt_AS_LONG(x);
-            if (is_unsigned && unlikely(val < 0)) {
-                goto raise_neg_overflow;
-            }
-            return (unsigned int) val;
-        }
-    } else
-#endif
-    if (likely(PyLong_Check(x))) {
-        if (is_unsigned) {
-#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3
- #if CYTHON_USE_PYLONG_INTERNALS
-            switch (Py_SIZE(x)) {
-                case  0: return 0;
-                case  1: __PYX_VERIFY_RETURN_INT(unsigned int, digit, ((PyLongObject*)x)->ob_digit[0]);
-            }
- #endif
-#endif
-            if (unlikely(Py_SIZE(x) < 0)) {
-                goto raise_neg_overflow;
-            }
-            if (sizeof(unsigned int) <= sizeof(unsigned long)) {
-                __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, PyLong_AsUnsignedLong(x))
-            } else if (sizeof(unsigned int) <= sizeof(unsigned long long)) {
-                __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long long, PyLong_AsUnsignedLongLong(x))
-            }
-        } else {
-#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3
- #if CYTHON_USE_PYLONG_INTERNALS
-            switch (Py_SIZE(x)) {
-                case  0: return 0;
-                case  1: __PYX_VERIFY_RETURN_INT(unsigned int,  digit, +(((PyLongObject*)x)->ob_digit[0]));
-                case -1: __PYX_VERIFY_RETURN_INT(unsigned int, sdigit, -(sdigit) ((PyLongObject*)x)->ob_digit[0]);
-            }
- #endif
-#endif
-            if (sizeof(unsigned int) <= sizeof(long)) {
-                __PYX_VERIFY_RETURN_INT(unsigned int, long, PyLong_AsLong(x))
-            } else if (sizeof(unsigned int) <= sizeof(long long)) {
-                __PYX_VERIFY_RETURN_INT(unsigned int, long long, PyLong_AsLongLong(x))
-            }
-        }
-        {
-#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray)
-            PyErr_SetString(PyExc_RuntimeError,
-                            "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers");
-#else
-            unsigned int val;
-            PyObject *v = __Pyx_PyNumber_Int(x);
- #if PY_MAJOR_VERSION < 3
-            if (likely(v) && !PyLong_Check(v)) {
-                PyObject *tmp = v;
-                v = PyNumber_Long(tmp);
-                Py_DECREF(tmp);
-            }
- #endif
-            if (likely(v)) {
-                int one = 1; int is_little = (int)*(unsigned char *)&one;
-                unsigned char *bytes = (unsigned char *)&val;
-                int ret = _PyLong_AsByteArray((PyLongObject *)v,
-                                              bytes, sizeof(val),
-                                              is_little, !is_unsigned);
-                Py_DECREF(v);
-                if (likely(!ret))
-                    return val;
-            }
-#endif
-            return (unsigned int) -1;
-        }
-    } else {
-        unsigned int val;
-        PyObject *tmp = __Pyx_PyNumber_Int(x);
-        if (!tmp) return (unsigned int) -1;
-        val = __Pyx_PyInt_As_unsigned_int(tmp);
-        Py_DECREF(tmp);
-        return val;
-    }
-raise_overflow:
-    PyErr_SetString(PyExc_OverflowError,
-        "value too large to convert to unsigned int");
-    return (unsigned int) -1;
-raise_neg_overflow:
-    PyErr_SetString(PyExc_OverflowError,
-        "can't convert negative value to unsigned int");
-    return (unsigned int) -1;
-}
-
 static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) {
     const int neg_one = (int) -1, const_zero = 0;
     const int is_unsigned = neg_one > const_zero;
@@ -32643,6 +32615,127 @@ bad:
     return module;
 }
 
+static CYTHON_INLINE unsigned int __Pyx_PyInt_As_unsigned_int(PyObject *x) {
+    const unsigned int neg_one = (unsigned int) -1, const_zero = 0;
+    const int is_unsigned = neg_one > const_zero;
+#if PY_MAJOR_VERSION < 3
+    if (likely(PyInt_Check(x))) {
+        if (sizeof(unsigned int) < sizeof(long)) {
+            __PYX_VERIFY_RETURN_INT(unsigned int, long, PyInt_AS_LONG(x))
+        } else {
+            long val = PyInt_AS_LONG(x);
+            if (is_unsigned && unlikely(val < 0)) {
+                goto raise_neg_overflow;
+            }
+            return (unsigned int) val;
+        }
+    } else
+#endif
+    if (likely(PyLong_Check(x))) {
+        if (is_unsigned) {
+#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3
+ #if CYTHON_USE_PYLONG_INTERNALS
+            switch (Py_SIZE(x)) {
+                case  0: return 0;
+                case  1: __PYX_VERIFY_RETURN_INT(unsigned int, digit, ((PyLongObject*)x)->ob_digit[0]);
+            }
+ #endif
+#endif
+            if (unlikely(Py_SIZE(x) < 0)) {
+                goto raise_neg_overflow;
+            }
+            if (sizeof(unsigned int) <= sizeof(unsigned long)) {
+                __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, PyLong_AsUnsignedLong(x))
+            } else if (sizeof(unsigned int) <= sizeof(unsigned long long)) {
+                __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long long, PyLong_AsUnsignedLongLong(x))
+            }
+        } else {
+#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3
+ #if CYTHON_USE_PYLONG_INTERNALS
+            switch (Py_SIZE(x)) {
+                case  0: return 0;
+                case  1: __PYX_VERIFY_RETURN_INT(unsigned int,  digit, +(((PyLongObject*)x)->ob_digit[0]));
+                case -1: __PYX_VERIFY_RETURN_INT(unsigned int, sdigit, -(sdigit) ((PyLongObject*)x)->ob_digit[0]);
+            }
+ #endif
+#endif
+            if (sizeof(unsigned int) <= sizeof(long)) {
+                __PYX_VERIFY_RETURN_INT(unsigned int, long, PyLong_AsLong(x))
+            } else if (sizeof(unsigned int) <= sizeof(long long)) {
+                __PYX_VERIFY_RETURN_INT(unsigned int, long long, PyLong_AsLongLong(x))
+            }
+        }
+        {
+#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray)
+            PyErr_SetString(PyExc_RuntimeError,
+                            "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers");
+#else
+            unsigned int val;
+            PyObject *v = __Pyx_PyNumber_Int(x);
+ #if PY_MAJOR_VERSION < 3
+            if (likely(v) && !PyLong_Check(v)) {
+                PyObject *tmp = v;
+                v = PyNumber_Long(tmp);
+                Py_DECREF(tmp);
+            }
+ #endif
+            if (likely(v)) {
+                int one = 1; int is_little = (int)*(unsigned char *)&one;
+                unsigned char *bytes = (unsigned char *)&val;
+                int ret = _PyLong_AsByteArray((PyLongObject *)v,
+                                              bytes, sizeof(val),
+                                              is_little, !is_unsigned);
+                Py_DECREF(v);
+                if (likely(!ret))
+                    return val;
+            }
+#endif
+            return (unsigned int) -1;
+        }
+    } else {
+        unsigned int val;
+        PyObject *tmp = __Pyx_PyNumber_Int(x);
+        if (!tmp) return (unsigned int) -1;
+        val = __Pyx_PyInt_As_unsigned_int(tmp);
+        Py_DECREF(tmp);
+        return val;
+    }
+raise_overflow:
+    PyErr_SetString(PyExc_OverflowError,
+        "value too large to convert to unsigned int");
+    return (unsigned int) -1;
+raise_neg_overflow:
+    PyErr_SetString(PyExc_OverflowError,
+        "can't convert negative value to unsigned int");
+    return (unsigned int) -1;
+}
+
+static CYTHON_INLINE PyObject* __Pyx_PyInt_From_unsigned_int(unsigned int value) {
+    const unsigned int neg_one = (unsigned int) -1, const_zero = 0;
+    const int is_unsigned = neg_one > const_zero;
+    if (is_unsigned) {
+        if (sizeof(unsigned int) < sizeof(long)) {
+            return PyInt_FromLong((long) value);
+        } else if (sizeof(unsigned int) <= sizeof(unsigned long)) {
+            return PyLong_FromUnsignedLong((unsigned long) value);
+        } else if (sizeof(unsigned int) <= sizeof(unsigned long long)) {
+            return PyLong_FromUnsignedLongLong((unsigned long long) value);
+        }
+    } else {
+        if (sizeof(unsigned int) <= sizeof(long)) {
+            return PyInt_FromLong((long) value);
+        } else if (sizeof(unsigned int) <= sizeof(long long)) {
+            return PyLong_FromLongLong((long long) value);
+        }
+    }
+    {
+        int one = 1; int little = (int)*(unsigned char *)&one;
+        unsigned char *bytes = (unsigned char *)&value;
+        return _PyLong_FromByteArray(bytes, sizeof(unsigned int),
+                                     little, !is_unsigned);
+    }
+}
+
 static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) {
     const int neg_one = (int) -1, const_zero = 0;
     const int is_unsigned = neg_one > const_zero;
diff --git a/python/cdec/lattice.pxd b/python/cdec/lattice.pxd
index 8ad710e5..ed290d5a 100644
--- a/python/cdec/lattice.pxd
+++ b/python/cdec/lattice.pxd
@@ -1,14 +1,14 @@
 from libcpp.vector cimport vector
 from libcpp.string cimport string
-from utils cimport WordID
+from utils cimport *
 
 cdef extern from "decoder/lattice.h":
     cdef cppclass LatticeArc:
         WordID label
-        double cost
+        FastSparseVector[double] features
         int dist2next
         LatticeArc()
-        LatticeArc(WordID w, double c, int i)
+        LatticeArc(WordID w, FastSparseVector[double]& features, int i)
 
     cdef cppclass Lattice(vector): # (vector[vector[LatticeArc]])
         Lattice()
diff --git a/python/cdec/lattice.pxi b/python/cdec/lattice.pxi
index 8000b61e..2564be8a 100644
--- a/python/cdec/lattice.pxi
+++ b/python/cdec/lattice.pxi
@@ -1,4 +1,5 @@
 cimport lattice
+from utils cimport *
 
 cdef class Lattice:
     cdef lattice.Lattice* lattice
@@ -31,16 +32,17 @@ cdef class Lattice:
         for i in range(arc_vector.size()):
             arc = &arc_vector[i]
             label = unicode(TDConvert(arc.label).c_str(), 'utf8')
-            arcs.append((label, arc.cost, arc.dist2next))
+            #arcs.append((label, arc.features, arc.dist2next))
+            arcs.append((label, arc.dist2next))
         return tuple(arcs)
 
     def __setitem__(self, int index, tuple arcs):
         if not 0 <= index < len(self):
             raise IndexError('lattice index out of range')
         cdef lattice.LatticeArc* arc
-        for (label, cost, dist2next) in arcs:
+        for (label, dist2next) in arcs:
             label_str = as_str(label)
-            arc = new lattice.LatticeArc(TDConvert(label_str), cost, dist2next)
+            arc = new lattice.LatticeArc() #TDConvert(label_str), features, dist2next)
             self.lattice[0][index].push_back(arc[0])
             del arc
 
diff --git a/python/cdec/sa/extract.py b/python/cdec/sa/extract.py
index b6c11f05..88ad2170 100644
--- a/python/cdec/sa/extract.py
+++ b/python/cdec/sa/extract.py
@@ -97,6 +97,57 @@ def stream_extract():
             sys.stdout.write('Error: see README.md for stream mode usage.  Skipping line: {}\n'.format(line.strip()))
         sys.stdout.flush()
 
+def stream_extract2(url):
+    global extractor, online, compress
+    import nanomsg
+    from nanomsg import Socket, PAIR, PUB
+    socket = nanomsg.Socket(nanomsg.PAIR)
+    socket.bind(url)
+    sys.stderr.write("[extractor] sending hello ...\n")
+    socket.send("hello")
+    default_context = "default_context"
+    while True:
+        line = socket.recv()
+        if line.strip() == "shutdown":
+            sys.stderr.write("[extractor] shutting down\n")
+            break
+        if not line:
+            break
+        fields = re.split('\s*\|\|\|\s*', line.strip())
+        # error
+        if len(fields) == 1:
+            socket.send("[extractor] error: can't process input '{}'".format(line.strip()))
+        # context ||| cmd
+        if len(fields) == 2:
+            (context, cmd) = fields
+            assert(context == default_context)
+            if cmd.lower() == 'drop':
+                if online:
+                    extractor.drop_ctx(context)
+                    socket.send("[extractor] dropping context '{}'".format(context))
+                else:
+                    socket.send("[extractor] error: online mode not set, skipping input: {}".format(line.strip()))
+            else:
+              socket.send("[extractor] don't know command '{}'".format(cmd))
+        # context ||| sentence ||| grammar_file
+        elif len(fields) == 3:
+            (context, sentence, grammar_file) = fields
+            assert(context == default_context)
+            with (gzip.open if compress else open)(grammar_file, 'w') as output:
+                for rule in extractor.grammar(sentence, context):
+                    output.write(str(rule)+'\n')
+            socket.send('{}'.format(grammar_file))
+        # context ||| sentence ||| reference ||| alignment
+        elif len(fields) == 4:
+            (context, sentence, reference, alignment) = fields
+            assert(context == default_context)
+            extractor.add_instance(sentence, reference, alignment, context)
+            socket.send("[extractor] learning (context: '{}')".format(context))
+        else:
+            socket.send("[extractor] error, skipping input: '{}'".format(line.strip()))
+    socket.send("off")
+    socket.close()
+
 def main():
     global online
     logging.basicConfig(level=logging.INFO)
@@ -117,9 +168,13 @@ def main():
                         help='compress grammars with gzip')
     parser.add_argument('-t', '--stream', action='store_true',
                         help='stream mode (see README.md)')
+    parser.add_argument('-u', '--stream2', action='store_true',
+                        help='stream2 mode')
+    parser.add_argument('-S', '--sock_url', default='tcp://127.0.0.1:8888',
+                        help='socket url')
     args = parser.parse_args()
 
-    if not (args.grammars or args.stream):
+    if not (args.grammars or (args.stream or args.stream2)):
         sys.stderr.write('Error: either -g/--grammars or -t/--stream required\n')
         sys.exit(1)
 
@@ -133,6 +188,9 @@ def main():
 
     online = args.online
     stream = args.stream
+    stream2 = args.stream2
+    if stream2:
+        online = True
 
     start_time = monitor_cpu()
     if args.jobs > 1:
@@ -150,6 +208,8 @@ def main():
         make_extractor(args)
         if stream:
             stream_extract()
+        if stream2:
+            stream_extract2(args.sock_url)
         else:
             for output in map(extract, enumerate(sys.stdin)):
                 print(output)
diff --git a/training/dtrain/Makefile.am b/training/dtrain/Makefile.am
index 82aac988..74c2a4b2 100644
--- a/training/dtrain/Makefile.am
+++ b/training/dtrain/Makefile.am
@@ -1,4 +1,4 @@
-bin_PROGRAMS = dtrain dtrain_net
+bin_PROGRAMS = dtrain dtrain_net dtrain_net_interface
 
 dtrain_SOURCES = dtrain.cc dtrain.h sample.h score.h update.h
 dtrain_LDADD   = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a
@@ -6,5 +6,8 @@ dtrain_LDADD   = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mte
 dtrain_net_SOURCES = dtrain_net.cc dtrain_net.h dtrain.h sample.h score.h update.h
 dtrain_net_LDADD   = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a /usr/local/lib/libnanomsg.so
 
-AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I /usr/local/include
+dtrain_net_interface_SOURCES = dtrain_net_interface.cc dtrain_net_interface.h dtrain.h sample_net_interface.h score_net_interface.h update.h
+dtrain_net_interface_LDFLAGS = -rdynamic
+dtrain_net_interface_LDADD   = ../../decoder/libcdec.a ../../klm/search/libksearch.a ../../mteval/libmteval.a ../../utils/libutils.a ../../klm/lm/libklm.a ../../klm/util/libklm_util.a ../../klm/util/double-conversion/libklm_util_double.a /srv/postedit/lib/nanomsg-0.5-beta/lib/libnanomsg.so
 
+AM_CPPFLAGS = -W -Wall -Wno-sign-compare -I$(top_srcdir)/utils -I$(top_srcdir)/decoder -I$(top_srcdir)/mteval -I/usr/local/include -I/srv/postedit/lib/cppnanomsg
diff --git a/training/dtrain/dtrain_net.cc b/training/dtrain/dtrain_net.cc
index 946b7587..306da957 100644
--- a/training/dtrain/dtrain_net.cc
+++ b/training/dtrain/dtrain_net.cc
@@ -67,16 +67,19 @@ main(int argc, char** argv)
       } else {
         vector<string> parts;
         boost::algorithm::split_regex(parts, in, boost::regex(" \\|\\|\\| "));
-        source = parts[0];
-        parts.erase(parts.begin());
-        for (auto s: parts) {
-          vector<WordID> r;
-          vector<string> toks;
-          boost::split(toks, s, boost::is_any_of(" "));
-          for (auto tok: toks)
-            r.push_back(TD::Convert(tok));
-          refs.emplace_back(MakeNgrams(r, N));
-          rsz.push_back(r.size());
+        if (parts[0] == "act:translate") {
+        } else {
+          source = parts[0];
+          parts.erase(parts.begin());
+          for (auto s: parts) {
+            vector<WordID> r;
+            vector<string> toks;
+            boost::split(toks, s, boost::is_any_of(" "));
+            for (auto tok: toks)
+              r.push_back(TD::Convert(tok));
+            refs.emplace_back(MakeNgrams(r, N));
+            rsz.push_back(r.size());
+          }
         }
       }
     }
diff --git a/training/dtrain/dtrain_net.h b/training/dtrain/dtrain_net.h
index 24f95500..e0d33d64 100644
--- a/training/dtrain/dtrain_net.h
+++ b/training/dtrain/dtrain_net.h
@@ -42,7 +42,9 @@ dtrain_net_init(int argc, char** argv, po::variables_map* conf)
     ("decoder_conf,C", po::value<string>(),                      "configuration file for decoder")
     ("k",              po::value<size_t>()->default_value(100),              "size of kbest list")
     ("N",              po::value<size_t>()->default_value(4),          "N for BLEU approximation")
-    ("margin,m",       po::value<weight_t>()->default_value(0.),   "margin for margin perceptron");
+    ("margin,m",       po::value<weight_t>()->default_value(0.),   "margin for margin perceptron")
+    ("output,o",       po::value<string>()->default_value(""),               "final weights file")
+    ("input_weights,w",   po::value<string>(),                               "input weights file");
   po::options_description cl("Command Line Options");
   cl.add_options()
     ("conf,c", po::value<string>(), "dtrain configuration file")
diff --git a/training/dtrain/dtrain_net_interface.cc b/training/dtrain/dtrain_net_interface.cc
new file mode 100644
index 00000000..37dff496
--- /dev/null
+++ b/training/dtrain/dtrain_net_interface.cc
@@ -0,0 +1,411 @@
+#include "dtrain_net_interface.h"
+#include "sample_net_interface.h"
+#include "score_net_interface.h"
+#include "update.h"
+
+#include <nanomsg/nn.h>
+#include <nanomsg/pair.h>
+#include "nn.hpp"
+
+#include <sys/types.h>  // mkfifo
+#include <sys/stat.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <fcntl.h>
+
+
+using namespace dtrain;
+
+int
+main(int argc, char** argv)
+{
+  // get configuration
+  po::variables_map conf;
+  if (!dtrain_net_init(argc, argv, &conf))
+    exit(1); // something is wrong
+  const size_t k              = conf["k"].as<size_t>();
+  const size_t N              = conf["N"].as<size_t>();
+  const weight_t margin       = conf["margin"].as<weight_t>();
+  const string master_addr    = conf["addr"].as<string>();
+  const string output_fn      = conf["output"].as<string>();
+  const string debug_fn       = conf["debug_output"].as<string>();
+  vector<string> dense_features;
+  boost::split(dense_features, conf["dense_features"].as<string>(),
+               boost::is_any_of(" "));
+  const bool output_derivation = conf["output_derivation"].as<bool>();
+  const bool output_rules      = conf["output_rules"].as<bool>();
+
+  // update lm
+  /*if (conf["update_lm_fn"].as<string>() != "")
+    mkfifo(conf["update_lm_fn"].as<string>().c_str(), 0666);*/
+
+  // setup socket
+  nn::socket sock(AF_SP, NN_PAIR);
+  sock.bind(master_addr.c_str());
+  string hello = "hello";
+  sock.send(hello.c_str(), hello.size()+1, 0);
+
+  // setup decoder
+  register_feature_functions();
+  SetSilent(true);
+  ReadFile f(conf["decoder_conf"].as<string>());
+  Decoder decoder(f.stream());
+  ScoredKbest* observer = new ScoredKbest(k, new PerSentenceBleuScorer(N));
+
+  // weights
+  vector<weight_t>& decoder_weights = decoder.CurrentWeightVector();
+  SparseVector<weight_t> lambdas, w_average, original_lambdas;
+  if (conf.count("input_weights")) {
+    Weights::InitFromFile(conf["input_weights"].as<string>(), &decoder_weights);
+    Weights::InitSparseVector(decoder_weights, &lambdas);
+    Weights::InitSparseVector(decoder_weights, &original_lambdas);
+  }
+
+  // learning rates
+  SparseVector<weight_t> learning_rates, original_learning_rates;
+  weight_t learning_rate_R, original_learning_rate_R;
+  weight_t learning_rate_RB, original_learning_rate_RB;
+  weight_t learning_rate_Shape, original_learning_rate_Shape;
+  vector<weight_t> l;
+  Weights::InitFromFile(conf["learning_rates"].as<string>(), &l);
+  Weights::InitSparseVector(l, &learning_rates);
+  original_learning_rates      = learning_rates;
+  learning_rate_R              = conf["learning_rate_R"].as<weight_t>();
+  original_learning_rate_R     = learning_rate_R;
+  learning_rate_RB             = conf["learning_rate_RB"].as<weight_t>();
+  original_learning_rate_RB    = learning_rate_RB;
+  learning_rate_Shape          = conf["learning_rate_Shape"].as<weight_t>();
+  original_learning_rate_Shape = learning_rate_Shape;
+
+  cerr << _p4;
+  // output configuration
+  cerr << "dtrain_net_interface" << endl << "Parameters:" << endl;
+  cerr << setw(25) << "k " << k << endl;
+  cerr << setw(25) << "N " << N << endl;
+  cerr << setw(25) << "margin " << margin << endl;
+  cerr << setw(25) << "decoder conf " << "'"
+       << conf["decoder_conf"].as<string>() << "'" << endl;
+  cerr << setw(25) << "output " << "'" <<  output_fn << "'" << endl;
+  cerr << setw(25) << "debug "  << "'" << debug_fn   << "'" << endl;
+  cerr << setw(25) << "learning rates "      << "'"
+       << conf["learning_rates"].as<string>() << "'" << endl;
+  cerr << setw(25) << "learning rate R "     << learning_rate_R     << endl;
+  cerr << setw(25) << "learning rate RB "    << learning_rate_RB    << endl;
+  cerr << setw(25) << "learning rate Shape " << learning_rate_Shape << endl;
+
+  // debug
+  ostringstream debug_output;
+
+  string done = "done";
+
+  vector<ScoredHyp>* samples;
+
+  size_t i = 0;
+  while(true)
+  {
+    cerr << "[dtrain] looping" << endl;
+    // debug --
+    debug_output.str(string());
+    debug_output.clear();
+    debug_output << "{" << endl; // hack us a nice JSON output
+    // -- debug
+
+    bool just_translate = false;
+
+    char *buf = NULL;
+    string source;
+    vector<Ngrams> refs;
+    vector<size_t> rsz;
+    bool next = true;
+    size_t sz = sock.recv(&buf, NN_MSG, 0);
+    if (buf) {
+      const string in(buf, buf+sz);
+      nn::freemsg(buf);
+      cerr << "[dtrain] got input '" << in << "'" << endl;
+      if (boost::starts_with(in, "set_learning_rates")) { // set learning rates
+        stringstream ss(in);
+        string _,name; weight_t w;
+        ss >> _; ss >> name; ss >> w;
+        weight_t before = 0;
+        ostringstream long_name;
+        if (name == "R") {
+          before = learning_rate_R;
+          learning_rate_R = w;
+          long_name << "rule id feature group";
+        } else if (name == "RB") {
+          before = learning_rate_RB;
+          learning_rate_RB = w;
+          long_name << "rule bigram feature group";
+        } else if (name == "Shape") {
+          before = learning_rate_Shape;
+          learning_rate_Shape = w;
+          long_name << "rule shape feature group";
+        } else {
+          unsigned fid = FD::Convert(name);
+          before = learning_rates[fid];
+          learning_rates[fid] = w;
+          long_name << "feature '" << name << "'";
+        }
+        ostringstream o;
+        o << "set learning rate for " << long_name.str() << " to " << w
+          << " (was: " << before << ")" << endl;
+        string s = o.str();
+        cerr << "[dtrain] " << s;
+        cerr << "[dtrain] done, looping again" << endl;
+        sock.send(s.c_str(), s.size()+1, 0);
+        continue;
+      } else if (boost::starts_with(in, "reset_learning_rates")) {
+        cerr << "[dtrain] resetting learning rates" << endl;
+        learning_rates = original_learning_rates;
+        learning_rate_R = original_learning_rate_R;
+        learning_rate_RB = original_learning_rate_RB;
+        learning_rate_Shape = original_learning_rate_Shape;
+        cerr << "[dtrain] done, looping again" << endl;
+        sock.send(done.c_str(), done.size()+1, 0);
+        continue;
+      } else if (boost::starts_with(in, "set_weights")) { // set learning rates
+        stringstream ss(in);
+        string _,name; weight_t w;
+        ss >> _; ss >> name; ss >> w;
+        weight_t before = 0;
+        ostringstream o;
+        unsigned fid = FD::Convert(name);
+        before = lambdas[fid];
+        lambdas[fid] = w;
+        o << "set weight for feature '" << name << "'"
+          << "' to " << w << " (was: " << before << ")" << endl;
+        string s = o.str();
+        cerr << "[dtrain] " << s;
+        cerr << "[dtrain] done, looping again" << endl;
+        sock.send(s.c_str(), s.size()+1, 0);
+        continue;
+      } else if (boost::starts_with(in, "reset_weights")) { // reset weights
+        cerr << "[dtrain] resetting weights" << endl;
+        lambdas = original_lambdas;
+        cerr << "[dtrain] done, looping again" << endl;
+        sock.send(done.c_str(), done.size()+1, 0);
+        continue;
+      } else if (in == "shutdown") { // shut down
+        cerr << "[dtrain] got shutdown signal" << endl;
+        next = false;
+        continue;
+      } else if (boost::starts_with(in, "get_weight")) { // get weight
+        stringstream ss(in);
+        string _,name;
+        ss >> _; ss >> name;
+        cerr << "[dtrain] getting weight for " << name << endl;
+        ostringstream o;
+        unsigned fid = FD::Convert(name);
+        weight_t w = lambdas[fid];
+        o << w;
+        string s = o.str();
+        sock.send(s.c_str(), s.size()+1, 0);
+        continue;
+      } else if (boost::starts_with(in, "get_rate")) { // get rate
+        stringstream ss(in);
+        string _,name;
+        ss >> _; ss >> name;
+        cerr << "[dtrain] getting rate for " << name << endl;
+        ostringstream o;
+        unsigned fid = FD::Convert(name);
+        weight_t  r;
+        if (name == "R")
+          r = learning_rate_R;
+        else if (name == "RB")
+          r = learning_rate_RB;
+        else if (name == "Shape")
+          r = learning_rate_Shape;
+        else
+          r = learning_rates[fid];
+        o << r;
+        string s = o.str();
+        sock.send(s.c_str(), s.size()+1, 0);
+        continue;
+      } else { // translate
+        vector<string> parts;
+        boost::algorithm::split_regex(parts, in, boost::regex(" \\|\\|\\| "));
+        if (parts[0] == "act:translate" || parts[0] == "act:translate_learn") {
+          if (parts[0] == "act:translate")
+            just_translate = true;
+          cerr << "[dtrain] translating ..." << endl;
+          lambdas.init_vector(&decoder_weights);
+          observer->dont_score = true;
+          decoder.Decode(parts[1], observer);
+          observer->dont_score = false;
+          samples = observer->GetSamples();
+          if (parts[0] == "act:translate") {
+            ostringstream os;
+            cerr << "[dtrain] 1best features " << (*samples)[0].f << endl;
+            if (output_derivation) {
+              os << observer->GetViterbiTreeStr() << endl;
+            } else {
+              PrintWordIDVec((*samples)[0].w, os);
+            }
+            if (output_rules) {
+              os << observer->GetViterbiRules() << endl;
+            }
+            sock.send(os.str().c_str(), os.str().size()+1, 0);
+            cerr << "[dtrain] done translating, looping again" << endl;
+          }
+        } //else { // learn
+        if (!just_translate) {
+          cerr << "[dtrain] learning ..." << endl;
+          source = parts[1];
+          // debug --
+          debug_output << "\"source\":\""
+                       << escapeJson(source.substr(source.find_first_of(">")+2, source.find_last_of(">")-6))
+                       <<  "\"," << endl;
+          debug_output << "\"target\":\"" << escapeJson(parts[2]) <<  "\"," << endl;
+          // -- debug
+          parts.erase(parts.begin());
+          parts.erase(parts.begin());
+          for (auto s: parts) {
+            vector<WordID> r;
+            vector<string> toks;
+            boost::split(toks, s, boost::is_any_of(" "));
+            for (auto tok: toks)
+              r.push_back(TD::Convert(tok));
+            refs.emplace_back(MakeNgrams(r, N));
+            rsz.push_back(r.size());
+          }
+
+          for (size_t r = 0; r < samples->size(); r++)
+            (*samples)[r].gold = observer->scorer_->Score((*samples)[r].w, refs, rsz);
+        //}
+        //}
+        }
+      }
+    }
+
+    if (!next)
+      break;
+
+    // decode
+    lambdas.init_vector(&decoder_weights);
+
+    // debug --)
+    ostringstream os;
+    PrintWordIDVec((*samples)[0].w, os);
+    debug_output << "\"1best\":\"" << escapeJson(os.str());
+    debug_output << "\"," << endl;
+    debug_output << "\"kbest\":[" << endl;
+    size_t h = 0;
+    for (auto s: *samples) {
+      debug_output << "\"" << s.gold << " ||| "
+                           << s.model << " ||| " << s.rank << " ||| ";
+      for (auto o: s.f)
+        debug_output << escapeJson(FD::Convert(o.first)) << "=" << o.second << " ";
+      debug_output << " ||| ";
+      ostringstream os;
+      PrintWordIDVec(s.w, os);
+      debug_output << escapeJson(os.str());
+      h += 1;
+      debug_output << "\"";
+      if (h < samples->size()) {
+        debug_output << ",";
+      }
+      debug_output << endl;
+    }
+
+    debug_output << "]," << endl;
+    debug_output << "\"samples_size\":" << samples->size() << "," << endl;
+    debug_output << "\"weights_before\":{" << endl;
+    sparseVectorToJson(lambdas, debug_output);
+    debug_output << "}," << endl;
+    // -- debug
+    //
+
+    // get pairs
+    SparseVector<weight_t> update;
+    size_t num_up = CollectUpdates(samples, update, margin);
+
+    // debug --
+    debug_output << "\"1best_features\":{";
+    sparseVectorToJson((*samples)[0].f, debug_output);
+    debug_output << "}," << endl;
+    debug_output << "\"update_raw\":{";
+    sparseVectorToJson(update, debug_output);
+    debug_output << "}," << endl;
+    // -- debug
+
+    // update
+    for (auto it: update) {
+      string fname = FD::Convert(it.first);
+      unsigned k = it.first;
+      weight_t v = it.second;
+      if (learning_rates.find(it.first) != learning_rates.end()) {
+        update[k] = learning_rates[k]*v;
+      } else {
+        if (boost::starts_with(fname, "R:")) {
+          update[k] = learning_rate_R*v;
+        } else if (boost::starts_with(fname, "RBS:") ||
+                   boost::starts_with(fname, "RBT:")) {
+          update[k] = learning_rate_RB*v;
+        } else if (boost::starts_with(fname, "Shape_")) {
+          update[k] = learning_rate_Shape*v;
+        }
+      }
+    }
+    if (!just_translate) {
+      lambdas += update;
+    } else {
+      i++;
+    }
+
+    // debug --
+    debug_output << "\"update\":{";
+    sparseVectorToJson(update, debug_output);
+    debug_output << "}," << endl;
+    debug_output << "\"num_up\":" << num_up << "," << endl;
+    debug_output << "\"updated_features\":" << update.size() << "," << endl;
+    debug_output << "\"learning_rate_R\":" << learning_rate_R << "," << endl;
+    debug_output << "\"learning_rate_RB\":" << learning_rate_R << "," << endl;
+    debug_output << "\"learning_rate_Shape\":" << learning_rate_R << "," << endl;
+    debug_output << "\"learning_rates\":{" << endl;
+    sparseVectorToJson(learning_rates, debug_output);
+    debug_output << "}," << endl;
+    debug_output << "\"best_match\":\"";
+    ostringstream ps;
+    PrintWordIDVec((*samples)[0].w, ps);
+    debug_output << escapeJson(ps.str());
+    debug_output << "\"," << endl;
+    debug_output << "\"best_match_score\":" << (*samples)[0].gold << "," << endl ;
+    // -- debug
+
+    // debug --
+    debug_output << "\"weights_after\":{" << endl;
+    sparseVectorToJson(lambdas, debug_output);
+    debug_output << "}" << endl;
+    debug_output << "}" << endl;
+    // -- debug
+
+    // debug --
+    WriteFile f(debug_fn);
+    f.get() << debug_output.str();
+    f.get() << std::flush;
+    // -- debug
+
+    // write current weights
+    if (!just_translate) {
+      lambdas.init_vector(decoder_weights);
+      ostringstream fn;
+      fn << output_fn << "." << i << ".gz";
+      Weights::WriteToFile(fn.str(), decoder_weights, true);
+    }
+
+    if (!just_translate) {
+      cerr << "[dtrain] done learning, looping again" << endl;
+      sock.send(done.c_str(), done.size()+1, 0);
+    }
+
+  } // input loop
+
+  string shutdown = "off";
+  sock.send(shutdown.c_str(), shutdown.size()+1, 0);
+
+  cerr << "[dtrain] shutting down, goodbye" << endl;
+
+  return 0;
+}
+
diff --git a/training/dtrain/dtrain_net_interface.h b/training/dtrain/dtrain_net_interface.h
new file mode 100644
index 00000000..91c2e538
--- /dev/null
+++ b/training/dtrain/dtrain_net_interface.h
@@ -0,0 +1,134 @@
+#ifndef _DTRAIN_NET_INTERFACE_H_
+#define _DTRAIN_NET_INTERFACE_H_
+
+#include "dtrain.h"
+
+namespace dtrain
+{
+
+/*
+ * source: http://stackoverflow.com/questions/7724448/\
+           simple-json-string-escape-for-c/33799784#33799784
+ *
+ */
+inline string
+escapeJson(const string& s) {
+  ostringstream o;
+  for (auto c = s.cbegin(); c != s.cend(); c++) {
+    switch (*c) {
+    case '"': o << "\\\""; break;
+    case '\\': o << "\\\\"; break;
+    case '\b': o << "\\b"; break;
+    case '\f': o << "\\f"; break;
+    case '\n': o << "\\n"; break;
+    case '\r': o << "\\r"; break;
+    case '\t': o << "\\t"; break;
+    default:
+      if ('\x00' <= *c && *c <= '\x1f') {
+        o << "\\u"
+          << std::hex << std::setw(4) << std::setfill('0') << (int)*c;
+      } else {
+        o << *c;
+      }
+    }
+  }
+  return o.str();
+}
+
+inline void
+sparseVectorToJson(SparseVector<weight_t>& w, ostringstream& os)
+{
+  vector<string> strs;
+  for (typename SparseVector<weight_t>::iterator it=w.begin(),e=w.end(); it!=e; ++it) {
+    ostringstream a;
+    a << "\"" << escapeJson(FD::Convert(it->first)) << "\":" << it->second;
+    strs.push_back(a.str());
+  }
+  for (vector<string>::const_iterator it=strs.begin(); it!=strs.end(); it++) {
+    os << *it;
+    if ((it+1) != strs.end())
+      os << ",";
+    os << endl;
+  }
+}
+
+template<typename T>
+inline void
+vectorAsString(SparseVector<T>& v, ostringstream& os)
+{
+  SparseVector<weight_t>::iterator it = v.begin();
+  for (; it != v.end(); ++it) {
+    os << FD::Convert(it->first) << "=" << it->second;
+    auto peek = it;
+    if (++peek != v.end())
+      os << " ";
+  }
+}
+
+template<typename T>
+inline void
+updateVectorFromString(string& s, SparseVector<T>& v)
+{
+  string buf;
+  istringstream ss;
+  while (ss >> buf) {
+    size_t p = buf.find_last_of("=");
+    istringstream c(buf.substr(p+1,buf.size()));
+    weight_t val;
+    c >> val;
+    v[FD::Convert(buf.substr(0,p))] = val;
+  }
+}
+
+bool
+dtrain_net_init(int argc, char** argv, po::variables_map* conf)
+{
+  po::options_description ini("Configuration File Options");
+  ini.add_options()
+    ("decoder_conf,C",         po::value<string>(),                          "configuration file for decoder")
+    ("k",                      po::value<size_t>()->default_value(100),                  "size of kbest list")
+    ("N",                      po::value<size_t>()->default_value(4),              "N for BLEU approximation")
+    ("margin,m",               po::value<weight_t>()->default_value(0.),       "margin for margin perceptron")
+    ("output,o",               po::value<string>()->default_value(""),                   "final weights file")
+    ("input_weights,w",        po::value<string>(),                                      "input weights file")
+    ("learning_rates,l",       po::value<string>(),                  "pre-defined learning rates per feature")
+    ("learning_rate_R",        po::value<weight_t>(),                    "learning rate for rule id features")
+    ("learning_rate_RB",       po::value<weight_t>(),                "learning rate for rule bigram features")
+    ("learning_rate_Shape",    po::value<weight_t>(),                      "learning rate for shape features")
+    ("output_derivation,E",    po::bool_switch()->default_value(false),  "output derivation, not viterbi str")
+    ("output_rules,R",         po::bool_switch()->default_value(false),                   "also output rules")
+    ("update_lm_fn",      po::value<string>()->default_value(""),                                      "TODO")
+    ("dense_features,D",       po::value<string>()->default_value("EgivenFCoherent SampleCountF CountEF MaxLexFgivenE MaxLexEgivenF IsSingletonF IsSingletonFE Glue WordPenalty PassThrough LanguageModel LanguageModel_OOV Shape_S01111_T11011 Shape_S11110_T11011 Shape_S11100_T11000 Shape_S01110_T01110 Shape_S01111_T01111 Shape_S01100_T11000 Shape_S10000_T10000 Shape_S11100_T11100 Shape_S11110_T11110 Shape_S11110_T11010 Shape_S01100_T11100 Shape_S01000_T01000 Shape_S01010_T01010 Shape_S01111_T01011 Shape_S01100_T01100 Shape_S01110_T11010 Shape_S11000_T11000 Shape_S11000_T01100 IsSupportedOnline NewRule KnownRule OOVFix"),
+                                                                                             "dense features")
+    ("debug_output,d",   po::value<string>()->default_value(""),                      "file for debug output");
+  po::options_description cl("Command Line Options");
+  cl.add_options()
+    ("conf,c", po::value<string>(), "dtrain configuration file")
+    ("addr,a", po::value<string>(),         "address of master");
+  cl.add(ini);
+  po::store(parse_command_line(argc, argv, cl), *conf);
+  if (conf->count("conf")) {
+    ifstream f((*conf)["conf"].as<string>().c_str());
+    po::store(po::parse_config_file(f, ini), *conf);
+  }
+  po::notify(*conf);
+  if (!conf->count("decoder_conf")) {
+    cerr << "Missing decoder configuration. Exiting." << endl;
+    return false;
+  }
+  if (!conf->count("learning_rates")) {
+    cerr << "Missing learning rates. Exiting." << endl;
+    return false;
+  }
+  if (!conf->count("addr")) {
+    cerr << "No master address given! Exiting." << endl;
+    return false;
+  }
+
+  return true;
+}
+
+} // namespace
+
+#endif
+
diff --git a/training/dtrain/feed.rb b/training/dtrain/feed.rb
deleted file mode 100755
index fe8dd509..00000000
--- a/training/dtrain/feed.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'nanomsg'
-
-port = ARGV[0]
-sock = NanoMsg::PairSocket.new
-addr = "tcp://127.0.0.1:#{port}"
-sock.bind addr
-
-puts sock.recv
-while true
-  line = STDIN.gets
-  if !line
-    sock.send 'shutdown'
-    break
-  end
-  sock.send line.strip
-  sleep 1
-  sock.recv
-  sock.send "a=1 b=2"
-end
-
diff --git a/training/dtrain/nn.hpp b/training/dtrain/nn.hpp
deleted file mode 100644
index 50b8304c..00000000
--- a/training/dtrain/nn.hpp
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
-    Copyright (c) 2013 250bpm s.r.o.
-
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"),
-    to deal in the Software without restriction, including without limitation
-    the rights to use, copy, modify, merge, publish, distribute, sublicense,
-    and/or sell copies of the Software, and to permit persons to whom
-    the Software is furnished to do so, subject to the following conditions:
-
-    The above copyright notice and this permission notice shall be included
-    in all copies or substantial portions of the Software.
-
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
-    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-    IN THE SOFTWARE.
-*/
-
-#ifndef NN_HPP_INCLUDED
-#define NN_HPP_INCLUDED
-
-#include <nanomsg/nn.h>
-
-#include <cassert>
-#include <cstring>
-#include <algorithm>
-#include <exception>
-
-#if defined __GNUC__
-#define nn_slow(x) __builtin_expect ((x), 0)
-#else
-#define nn_slow(x) (x)
-#endif
-
-namespace nn
-{
-
-    class exception : public std::exception
-    {
-    public:
-
-        exception () : err (nn_errno ()) {}
-
-        virtual const char *what () const throw ()
-        {
-            return nn_strerror (err);
-        }
-
-        int num () const
-        {
-            return err;
-        }
-
-    private:
-
-        int err;
-    };
-
-    inline const char *symbol (int i, int *value)
-    {
-        return nn_symbol (i, value);
-    }
-
-    inline void *allocmsg (size_t size, int type)
-    {
-        void *msg = nn_allocmsg (size, type);
-        if (nn_slow (!msg))
-            throw nn::exception ();
-        return msg;
-    }
-
-    inline int freemsg (void *msg)
-    {
-        int rc = nn_freemsg (msg);
-        if (nn_slow (rc != 0))
-            throw nn::exception ();
-        return rc;
-    }
-
-    class socket
-    {
-    public:
-
-        inline socket (int domain, int protocol)
-        {
-            s = nn_socket (domain, protocol);
-            if (nn_slow (s < 0))
-                throw nn::exception ();
-        }
-
-        inline ~socket ()
-        {
-            int rc = nn_close (s);
-            assert (rc == 0);
-        }
-
-        inline void setsockopt (int level, int option, const void *optval,
-            size_t optvallen)
-        {
-            int rc = nn_setsockopt (s, level, option, optval, optvallen);
-            if (nn_slow (rc != 0))
-                throw nn::exception ();
-        }
-
-        inline void getsockopt (int level, int option, void *optval,
-            size_t *optvallen)
-        {
-            int rc = nn_getsockopt (s, level, option, optval, optvallen);
-            if (nn_slow (rc != 0))
-                throw nn::exception ();
-        }
-
-        inline int bind (const char *addr)
-        {
-            int rc = nn_bind (s, addr);
-            if (nn_slow (rc < 0))
-                throw nn::exception ();
-            return rc;
-        }
-
-        inline int connect (const char *addr)
-        {
-            int rc = nn_connect (s, addr);
-            if (nn_slow (rc < 0))
-                throw nn::exception ();
-            return rc;
-        }
-
-        inline void shutdown (int how)
-        {
-            int rc = nn_shutdown (s, how);
-            if (nn_slow (rc != 0))
-                throw nn::exception ();
-        }
-
-        inline int send (const void *buf, size_t len, int flags)
-        {
-            int rc = nn_send (s, buf, len, flags);
-            if (nn_slow (rc < 0)) {
-                if (nn_slow (nn_errno () != EAGAIN))
-                    throw nn::exception ();
-                return -1;
-            }
-            return rc;
-        }
-
-        inline int recv (void *buf, size_t len, int flags)
-        {
-            int rc = nn_recv (s, buf, len, flags);
-            if (nn_slow (rc < 0)) {
-                if (nn_slow (nn_errno () != EAGAIN))
-                    throw nn::exception ();
-                return -1;
-            }
-            return rc;
-        }
-
-        inline int sendmsg (const struct nn_msghdr *msghdr, int flags)
-        {
-            int rc = nn_sendmsg (s, msghdr, flags);
-            if (nn_slow (rc < 0)) {
-                if (nn_slow (nn_errno () != EAGAIN))
-                    throw nn::exception ();
-                return -1;
-            }
-            return rc;
-        }
-
-        inline int recvmsg (struct nn_msghdr *msghdr, int flags)
-        {
-            int rc = nn_recvmsg (s, msghdr, flags);
-            if (nn_slow (rc < 0)) {
-                if (nn_slow (nn_errno () != EAGAIN))
-                    throw nn::exception ();
-                return -1;
-            }
-            return rc;
-        }
-
-    private:
-
-        int s;
-
-        /*  Prevent making copies of the socket by accident. */
-        socket (const socket&);
-        void operator = (const socket&);
-    };
-
-    inline void term ()
-    {
-        nn_term ();
-    }
-
-}
-
-#undef nn_slow
-
-#endif
-
-
diff --git a/training/dtrain/sample.h b/training/dtrain/sample.h
index 03cc82c3..e24b65cf 100644
--- a/training/dtrain/sample.h
+++ b/training/dtrain/sample.h
@@ -16,6 +16,7 @@ struct ScoredKbest : public DecoderObserver
   PerSentenceBleuScorer* scorer_;
   vector<Ngrams>* ref_ngs_;
   vector<size_t>* ref_ls_;
+  string viterbi_tree_str;
 
   ScoredKbest(const size_t k, PerSentenceBleuScorer* scorer) :
     k_(k), scorer_(scorer) {}
@@ -40,6 +41,7 @@ struct ScoredKbest : public DecoderObserver
       samples_.push_back(h);
       effective_sz_++;
       feature_count_ += h.f.size();
+      viterbi_tree_str = hg->show_viterbi_tree(false);
     }
   }
 
@@ -51,6 +53,7 @@ struct ScoredKbest : public DecoderObserver
   }
   inline size_t GetFeatureCount() { return feature_count_; }
   inline size_t GetSize() { return effective_sz_; }
+  inline string GetViterbiTreeString() { return viterbi_tree_str; }
 };
 
 } // namespace
diff --git a/training/dtrain/sample_net_interface.h b/training/dtrain/sample_net_interface.h
new file mode 100644
index 00000000..6d00e5d5
--- /dev/null
+++ b/training/dtrain/sample_net_interface.h
@@ -0,0 +1,68 @@
+#ifndef _DTRAIN_SAMPLE_NET_H_
+#define _DTRAIN_SAMPLE_NET_H_
+
+#include "kbest.h"
+
+#include "score_net_interface.h"
+
+namespace dtrain
+{
+
+struct ScoredKbest : public DecoderObserver
+{
+  const size_t k_;
+  size_t feature_count_, effective_sz_;
+  vector<ScoredHyp> samples_;
+  PerSentenceBleuScorer* scorer_;
+  vector<Ngrams>* ref_ngs_;
+  vector<size_t>* ref_ls_;
+  bool dont_score;
+  string viterbiTreeStr_, viterbiRules_;
+
+  ScoredKbest(const size_t k, PerSentenceBleuScorer* scorer) :
+    k_(k), scorer_(scorer), dont_score(false) {}
+
+  virtual void
+  NotifyTranslationForest(const SentenceMetadata& /*smeta*/, Hypergraph* hg)
+  {
+    samples_.clear(); effective_sz_ = feature_count_ = 0;
+    KBest::KBestDerivations<vector<WordID>, ESentenceTraversal,
+      KBest::FilterUnique, prob_t, EdgeProb> kbest(*hg, k_);
+    for (size_t i = 0; i < k_; ++i) {
+      const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal,
+            KBest::FilterUnique, prob_t, EdgeProb>::Derivation* d =
+              kbest.LazyKthBest(hg->nodes_.size() - 1, i);
+      if (!d) break;
+      ScoredHyp h;
+      h.w = d->yield;
+      h.f = d->feature_values;
+      h.model = log(d->score);
+      h.rank = i;
+      if (!dont_score)
+        h.gold = scorer_->Score(h.w, *ref_ngs_, *ref_ls_);
+      samples_.push_back(h);
+      effective_sz_++;
+      feature_count_ += h.f.size();
+      viterbiTreeStr_ = hg->show_viterbi_tree(false);
+      ostringstream ss;
+      ViterbiRules(*hg, &ss);
+      viterbiRules_ = ss.str();
+    }
+  }
+
+  vector<ScoredHyp>* GetSamples() { return &samples_; }
+  inline void SetReference(vector<Ngrams>& ngs, vector<size_t>& ls)
+  {
+    ref_ngs_ = &ngs;
+    ref_ls_ = &ls;
+  }
+  inline size_t GetFeatureCount() { return feature_count_; }
+  inline size_t GetSize() { return effective_sz_; }
+  inline string GetViterbiTreeStr() { return viterbiTreeStr_; }
+  inline string GetViterbiRules() { return viterbiRules_; }
+};
+
+} // namespace
+
+#endif
+
diff --git a/training/dtrain/score.h b/training/dtrain/score.h
index 06dbc5a4..e6e60acb 100644
--- a/training/dtrain/score.h
+++ b/training/dtrain/score.h
@@ -153,7 +153,7 @@ struct PerSentenceBleuScorer
       size_t best = numeric_limits<size_t>::max();
       for (auto l: ref_ls) {
         size_t d = abs(hl-l);
-        if (d < best) { 
+        if (d < best) {
           best_idx = i;
           best = d;
         }
diff --git a/training/dtrain/score_net_interface.h b/training/dtrain/score_net_interface.h
new file mode 100644
index 00000000..58357cf6
--- /dev/null
+++ b/training/dtrain/score_net_interface.h
@@ -0,0 +1,200 @@
+#ifndef _DTRAIN_SCORE_NET_INTERFACE_H_
+#define _DTRAIN_SCORE_NET_INTERFACE_H_
+
+#include "dtrain.h"
+
+namespace dtrain
+{
+
+struct NgramCounts
+{
+  size_t N_;
+  map<size_t, weight_t> clipped_;
+  map<size_t, weight_t> sum_;
+
+  NgramCounts(const size_t N) : N_(N) { Zero(); }
+
+  inline void
+  operator+=(const NgramCounts& rhs)
+  {
+    if (rhs.N_ > N_) Resize(rhs.N_);
+    for (size_t i = 0; i < N_; i++) {
+      this->clipped_[i] += rhs.clipped_.find(i)->second;
+      this->sum_[i] += rhs.sum_.find(i)->second;
+    }
+  }
+
+  inline const NgramCounts
+  operator+(const NgramCounts &other) const
+  {
+    NgramCounts result = *this;
+    result += other;
+
+    return result;
+  }
+
+  inline void
+  Add(const size_t count, const size_t ref_count, const size_t i)
+  {
+    assert(i < N_);
+    if (count > ref_count) {
+      clipped_[i] += ref_count;
+    } else {
+      clipped_[i] += count;
+    }
+    sum_[i] += count;
+  }
+
+  inline void
+  Zero()
+  {
+    for (size_t i = 0; i < N_; i++) {
+      clipped_[i] = 0.;
+      sum_[i] = 0.;
+    }
+  }
+
+  inline void
+  Resize(size_t N)
+  {
+    if (N == N_) return;
+    else if (N > N_) {
+      for (size_t i = N_; i < N; i++) {
+        clipped_[i] = 0.;
+        sum_[i] = 0.;
+      }
+    } else { // N < N_
+      for (size_t i = N_-1; i > N-1; i--) {
+        clipped_.erase(i);
+        sum_.erase(i);
+      }
+    }
+    N_ = N;
+  }
+};
+
+typedef map<vector<WordID>, size_t> Ngrams;
+
+inline Ngrams
+MakeNgrams(const vector<WordID>& s, const size_t N)
+{
+  Ngrams ngrams;
+  vector<WordID> ng;
+  for (size_t i = 0; i < s.size(); i++) {
+    ng.clear();
+    for (size_t j = i; j < min(i+N, s.size()); j++) {
+      ng.push_back(s[j]);
+      ngrams[ng]++;
+    }
+  }
+
+  return ngrams;
+}
+
+inline NgramCounts
+MakeNgramCounts(const vector<WordID>& hyp,
+                const vector<Ngrams>& ref,
+                const size_t N)
+{
+  Ngrams hyp_ngrams = MakeNgrams(hyp, N);
+  NgramCounts counts(N);
+  Ngrams::iterator it, ti;
+  for (it = hyp_ngrams.begin(); it != hyp_ngrams.end(); it++) {
+    size_t max_ref_count = 0;
+    for (auto r: ref) {
+      ti = r.find(it->first);
+      if (ti != r.end())
+        max_ref_count = max(max_ref_count, ti->second);
+    }
+    counts.Add(it->second, min(it->second, max_ref_count), it->first.size()-1);
+  }
+
+  return counts;
+}
+
+/*
+ * per-sentence BLEU
+ * as in "Optimizing for Sentence-Level BLEU+1
+ *        Yields Short Translations"
+ * (Nakov et al. '12)
+ *
+ * [simply add 1 to reference length for calculation of BP]
+ *
+ */
+struct PerSentenceBleuScorer
+{
+  const size_t     N_;
+  vector<weight_t> w_;
+
+  PerSentenceBleuScorer(size_t n) : N_(n)
+  {
+    for (size_t i = 1; i <= N_; i++)
+      w_.push_back(1.0/N_);
+  }
+
+  inline weight_t
+  BrevityPenalty(const size_t hl, const size_t rl)
+  {
+    if (hl > rl)
+      return 1;
+
+    return exp(1 - (weight_t)rl/hl);
+  }
+
+  inline size_t
+  BestMatchLength(const size_t hl,
+                  const vector<size_t>& ref_ls)
+  {
+    size_t m;
+    if (ref_ls.size() == 1)  {
+      m = ref_ls.front();
+    } else {
+      size_t i = 0, best_idx = 0;
+      size_t best = numeric_limits<size_t>::max();
+      for (auto l: ref_ls) {
+        size_t d = abs(hl-l);
+        if (d < best) {
+          best_idx = i;
+          best = d;
+        }
+        i += 1;
+      }
+      m = ref_ls[best_idx];
+    }
+
+    return m;
+  }
+
+  weight_t
+  Score(const vector<WordID>& hyp,
+        const vector<Ngrams>& ref_ngs,
+        const vector<size_t>& ref_ls)
+  {
+    size_t hl = hyp.size(), rl = 0;
+    if (hl == 0) return 0.;
+    rl = BestMatchLength(hl, ref_ls);
+    if (rl == 0) return 0.;
+    NgramCounts counts = MakeNgramCounts(hyp, ref_ngs, N_);
+    size_t M = N_;
+    vector<weight_t> v = w_;
+    if (rl < N_) {
+      M = rl;
+      for (size_t i = 0; i < M; i++) v[i] = 1/((weight_t)M);
+    }
+    weight_t sum = 0, add = 0;
+    for (size_t i = 0; i < M; i++) {
+      if (i == 0 && (counts.sum_[i] == 0 || counts.clipped_[i] == 0)) return 0.;
+      if (i > 0) add = 1;
+      sum += v[i] * log(((weight_t)counts.clipped_[i] + add)
+                        / ((counts.sum_[i] + add)));
+    }
+
+    //return  BrevityPenalty(hl, rl+1) * exp(sum);
+    return  BrevityPenalty(hl, rl) * exp(sum);
+  }
+};
+
+} // namespace
+
+#endif
+
diff --git a/utils/Makefile.am b/utils/Makefile.am
index c858ac7e..ceffd5d6 100644
--- a/utils/Makefile.am
+++ b/utils/Makefile.am
@@ -1,4 +1,4 @@
-bin_PROGRAMS = reconstruct_weights atools dedup_corpus
+bin_PROGRAMS = reconstruct_weights atools atools_net dedup_corpus
 
 noinst_PROGRAMS = \
   ts \
@@ -98,6 +98,10 @@ atools_SOURCES = atools.cc
 atools_LDADD = libutils.a
 atools_LDFLAGS = $(STATIC_FLAGS)
 
+atools_net_SOURCES = atools_net.cc nn.hpp
+atools_net_LDADD = libutils.a
+atools_net_LDFLAGS = $(STATIC_FLAGS) /srv/postedit/lib/nanomsg-0.5-beta/lib/libnanomsg.so
+
 dedup_corpus_SOURCES = dedup_corpus.cc
 dedup_corpus_LDADD = libutils.a
 dedup_corpus_LDFLAGS = $(STATIC_FLAGS)
@@ -123,5 +127,5 @@ stringlib_test_LDADD = libutils.a $(BOOST_UNIT_TEST_FRAMEWORK_LDFLAGS) $(BOOST_U
 
 ################################################################
 # do NOT NOT NOT add any other -I includes NO NO NO NO NO ######
-AM_CPPFLAGS = -DBOOST_TEST_DYN_LINK -W -Wall -I. -I$(top_srcdir) -DTEST_DATA=\"$(top_srcdir)/utils/test_data\"
+AM_CPPFLAGS = -DBOOST_TEST_DYN_LINK -W -Wall -I. -I$(top_srcdir) -DTEST_DATA=\"$(top_srcdir)/utils/test_data\" -I/srv/postedit/lib/nanomsg-0.5-beta/include -I/srv/postedit/lib/cppnanomsg
 ################################################################
diff --git a/utils/atools_net.cc b/utils/atools_net.cc
new file mode 100644
index 00000000..fa5dfb96
--- /dev/null
+++ b/utils/atools_net.cc
@@ -0,0 +1,363 @@
+#include <iostream>
+#include <sstream>
+#include <vector>
+
+#include <queue>
+#include <map>
+#include <boost/program_options.hpp>
+#include <boost/shared_ptr.hpp>
+
+#include "filelib.h"
+#include "alignment_io.h"
+
+#include <boost/algorithm/string.hpp>
+#include <boost/regex.hpp>
+#include <boost/algorithm/string/regex.hpp>
+
+#include <nanomsg/nn.h>
+#include <nanomsg/pair.h>
+#include "nn.hpp"
+
+namespace po = boost::program_options;
+using namespace std;
+
+struct Command {
+  virtual ~Command() {}
+  virtual string Name() const = 0;
+
+  // returns 1 for alignment grid output [default]
+  // returns 2 if Summary() should be called [for AER, etc]
+  virtual int Result() const { return 1; }
+
+  virtual bool RequiresTwoOperands() const { return true; }
+  virtual void Apply(const Array2D<bool>& a, const Array2D<bool>& b, Array2D<bool>* x) = 0;
+  void EnsureSize(const Array2D<bool>& a, const Array2D<bool>& b, Array2D<bool>* x) {
+    x->resize(max(a.width(), b.width()), max(a.height(), b.height()));
+  }
+  static bool Safe(const Array2D<bool>& a, int i, int j) {
+    if (i >= 0 && j >= 0 && i < static_cast<int>(a.width()) && j < static_cast<int>(a.height()))
+      return a(i,j);
+    else
+      return false;
+  }
+  virtual void Summary() { assert(!"Summary should have been overridden"); }
+};
+
+// compute fmeasure, second alignment is reference, first is hyp
+struct FMeasureCommand : public Command {
+  FMeasureCommand() : matches(), num_predicted(), num_in_ref() {}
+  int Result() const { return 2; }
+  string Name() const { return "fmeasure"; }
+  bool RequiresTwoOperands() const { return true; }
+  void Apply(const Array2D<bool>& hyp, const Array2D<bool>& ref, Array2D<bool>* x) {
+    (void) x;   // AER just computes statistics, not an alignment
+    unsigned i_len = ref.width();
+    unsigned j_len = ref.height();
+    for (unsigned i = 0; i < i_len; ++i) {
+      for (unsigned j = 0; j < j_len; ++j) {
+        if (ref(i,j)) {
+          ++num_in_ref;
+          if (Safe(hyp, i, j)) ++matches;
+        }
+      }
+    }
+    for (unsigned i = 0; i < hyp.width(); ++i)
+      for (unsigned j = 0; j < hyp.height(); ++j)
+        if (hyp(i,j)) ++num_predicted;
+  }
+  void Summary() {
+    if (num_predicted == 0 || num_in_ref == 0) {
+      cerr << "Insufficient statistics to compute f-measure!\n";
+      abort();
+    }
+    const double prec = static_cast<double>(matches) / num_predicted;
+    const double rec = static_cast<double>(matches) / num_in_ref;
+    cout << "P: " << prec << endl;
+    cout << "R: " << rec << endl;
+    const double f = (2.0 * prec * rec) / (rec + prec);
+    cout << "F: " << f << endl;
+  }
+  int matches;
+  int num_predicted;
+  int num_in_ref;
+};
+
+struct DisplayCommand : public Command {
+  string Name() const { return "display"; }
+  bool RequiresTwoOperands() const { return false; }
+  void Apply(const Array2D<bool>& in, const Array2D<bool>&, Array2D<bool>* x) {
+    *x = in;
+    cout << *x << endl;
+  }
+};
+
+struct ConvertCommand : public Command {
+  string Name() const { return "convert"; }
+  bool RequiresTwoOperands() const { return false; }
+  void Apply(const Array2D<bool>& in, const Array2D<bool>&, Array2D<bool>* x) {
+    *x = in;
+  }
+};
+
+struct InvertCommand : public Command {
+  string Name() const { return "invert"; }
+  bool RequiresTwoOperands() const { return false; }
+  void Apply(const Array2D<bool>& in, const Array2D<bool>&, Array2D<bool>* x) {
+    Array2D<bool>& res = *x;
+    res.resize(in.height(), in.width());
+    for (unsigned i = 0; i < in.height(); ++i)
+      for (unsigned j = 0; j < in.width(); ++j)
+        res(i, j) = in(j, i);
+  }
+};
+
+struct IntersectCommand : public Command {
+  string Name() const { return "intersect"; }
+  bool RequiresTwoOperands() const { return true; }
+  void Apply(const Array2D<bool>& a, const Array2D<bool>& b, Array2D<bool>* x) {
+    EnsureSize(a, b, x);
+    Array2D<bool>& res = *x;
+    for (unsigned i = 0; i < a.width(); ++i)
+      for (unsigned j = 0; j < a.height(); ++j)
+        res(i, j) = Safe(a, i, j) && Safe(b, i, j);
+  }
+};
+
+struct UnionCommand : public Command {
+  string Name() const { return "union"; }
+  bool RequiresTwoOperands() const { return true; }
+  void Apply(const Array2D<bool>& a, const Array2D<bool>& b, Array2D<bool>* x) {
+    EnsureSize(a, b, x);
+    Array2D<bool>& res = *x;
+    for (unsigned i = 0; i < res.width(); ++i)
+      for (unsigned j = 0; j < res.height(); ++j)
+        res(i, j) = Safe(a, i, j) || Safe(b, i, j);
+  }
+};
+
+struct RefineCommand : public Command {
+  RefineCommand() {
+    neighbors_.push_back(make_pair(1,0));
+    neighbors_.push_back(make_pair(-1,0));
+    neighbors_.push_back(make_pair(0,1));
+    neighbors_.push_back(make_pair(0,-1));
+  }
+  bool RequiresTwoOperands() const { return true; }
+
+  void Align(unsigned i, unsigned j) {
+    res_(i, j) = true;
+    is_i_aligned_[i] = true;
+    is_j_aligned_[j] = true;
+  }
+
+  bool IsNeighborAligned(int i, int j) const {
+    for (unsigned k = 0; k < neighbors_.size(); ++k) {
+      const int di = neighbors_[k].first;
+      const int dj = neighbors_[k].second;
+      if (Safe(res_, i + di, j + dj))
+        return true;
+    }
+    return false;
+  }
+
+  bool IsNeitherAligned(int i, int j) const {
+    return !(is_i_aligned_[i] || is_j_aligned_[j]);
+  }
+
+  bool IsOneOrBothUnaligned(int i, int j) const {
+    return !(is_i_aligned_[i] && is_j_aligned_[j]);
+  }
+
+  bool KoehnAligned(int i, int j) const {
+    return IsOneOrBothUnaligned(i, j) && IsNeighborAligned(i, j);
+  }
+
+  typedef bool (RefineCommand::*Predicate)(int i, int j) const;
+
+ protected:
+  void InitRefine(
+      const Array2D<bool>& a,
+      const Array2D<bool>& b) {
+    res_.clear();
+    EnsureSize(a, b, &res_);
+    in_.clear(); un_.clear(); is_i_aligned_.clear(); is_j_aligned_.clear();
+    EnsureSize(a, b, &in_);
+    EnsureSize(a, b, &un_);
+    is_i_aligned_.resize(res_.width(), false);
+    is_j_aligned_.resize(res_.height(), false);
+    for (unsigned i = 0; i < in_.width(); ++i)
+      for (unsigned j = 0; j < in_.height(); ++j) {
+        un_(i, j) = Safe(a, i, j) || Safe(b, i, j);
+        in_(i, j) = Safe(a, i, j) && Safe(b, i, j);
+        if (in_(i, j)) Align(i, j);
+    }
+  }
+  // "grow" the resulting alignment using the points in adds
+  // if they match the constraints determined by pred
+  void Grow(Predicate pred, bool idempotent, const Array2D<bool>& adds) {
+    if (idempotent) {
+      for (unsigned i = 0; i < adds.width(); ++i)
+        for (unsigned j = 0; j < adds.height(); ++j) {
+          if (adds(i, j) && !res_(i, j) &&
+              (this->*pred)(i, j)) Align(i, j);
+        }
+      return;
+    }
+    set<pair<int, int> > p;
+    for (unsigned i = 0; i < adds.width(); ++i)
+      for (unsigned j = 0; j < adds.height(); ++j)
+        if (adds(i, j) && !res_(i, j))
+          p.insert(make_pair(i, j));
+    bool keep_going = !p.empty();
+    while (keep_going) {
+      keep_going = false;
+      set<pair<int, int> > added;
+      for (set<pair<int, int> >::iterator pi = p.begin(); pi != p.end(); ++pi) {
+        if ((this->*pred)(pi->first, pi->second)) {
+          Align(pi->first, pi->second);
+          added.insert(make_pair(pi->first, pi->second));
+          keep_going = true;
+        }
+      }
+      for (set<pair<int, int> >::iterator ai = added.begin(); ai != added.end(); ++ai)
+        p.erase(*ai);
+    }
+  }
+  Array2D<bool> res_;  // refined alignment
+  Array2D<bool> in_;   // intersection alignment
+  Array2D<bool> un_;   // union alignment
+  vector<bool> is_i_aligned_;
+  vector<bool> is_j_aligned_;
+  vector<pair<int,int> > neighbors_;
+};
+
+struct DiagCommand : public RefineCommand {
+  DiagCommand() {
+    neighbors_.push_back(make_pair(1,1));
+    neighbors_.push_back(make_pair(-1,1));
+    neighbors_.push_back(make_pair(1,-1));
+    neighbors_.push_back(make_pair(-1,-1));
+  }
+};
+
+struct GDCommand : public DiagCommand {
+  string Name() const { return "grow-diag"; }
+  void Apply(const Array2D<bool>& a, const Array2D<bool>& b, Array2D<bool>* x) {
+    InitRefine(a, b);
+    Grow(&RefineCommand::KoehnAligned, false, un_);
+    *x = res_;
+  }
+};
+
+struct GDFCommand : public DiagCommand {
+  string Name() const { return "grow-diag-final"; }
+  void Apply(const Array2D<bool>& a, const Array2D<bool>& b, Array2D<bool>* x) {
+    InitRefine(a, b);
+    Grow(&RefineCommand::KoehnAligned, false, un_);
+    Grow(&RefineCommand::IsOneOrBothUnaligned, true, a);
+    Grow(&RefineCommand::IsOneOrBothUnaligned, true, b);
+    *x = res_;
+  }
+};
+
+struct GDFACommand : public DiagCommand {
+  string Name() const { return "grow-diag-final-and"; }
+  void Apply(const Array2D<bool>& a, const Array2D<bool>& b, Array2D<bool>* x) {
+    InitRefine(a, b);
+    Grow(&RefineCommand::KoehnAligned, false, un_);
+    Grow(&RefineCommand::IsNeitherAligned, true, a);
+    Grow(&RefineCommand::IsNeitherAligned, true, b);
+    *x = res_;
+  }
+};
+
+map<string, boost::shared_ptr<Command> > commands;
+
+void InitCommandLine(unsigned argc, char** argv, po::variables_map* conf) {
+  po::options_description opts("Configuration options");
+  ostringstream os;
+  os << "Operation to perform:";
+  for (map<string, boost::shared_ptr<Command> >::iterator it = commands.begin();
+       it != commands.end(); ++it) {
+    os << ' ' << it->first;
+  }
+  string cstr = os.str();
+  opts.add_options()
+        ("command,c", po::value<string>()->default_value("convert"), cstr.c_str())
+        ("sock_url,S", po::value<string>()->default_value("tcp://127.0.0.1:60665"))
+        ("help,h", "Print this help message and exit");
+  po::options_description clo("Command line options");
+  po::options_description dcmdline_options;
+  dcmdline_options.add(opts);
+
+  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
+  po::notify(*conf);
+
+  if (conf->count("help") || conf->count("command") == 0) {
+    cerr << dcmdline_options << endl;
+    exit(1);
+  }
+  const string cmd = (*conf)["command"].as<string>();
+  if (commands.count(cmd) == 0) {
+    cerr << "Don't understand command: " << cmd << endl;
+    exit(1);
+  }
+}
+
+template<class C> static void AddCommand() {
+  C* c = new C;
+  commands[c->Name()].reset(c);
+}
+
+int main(int argc, char **argv) {
+  AddCommand<ConvertCommand>();
+  AddCommand<DisplayCommand>();
+  AddCommand<InvertCommand>();
+  AddCommand<IntersectCommand>();
+  AddCommand<UnionCommand>();
+  AddCommand<GDCommand>();
+  AddCommand<GDFCommand>();
+  AddCommand<GDFACommand>();
+  AddCommand<FMeasureCommand>();
+  po::variables_map conf;
+  InitCommandLine(argc, argv, &conf);
+  Command& cmd = *commands[conf["command"].as<string>()];
+
+  nn::socket sock(AF_SP, NN_PAIR);
+  string url = conf["sock_url"].as<string>();
+  sock.bind(url.c_str());
+  int to = 100;
+  sock.setsockopt(NN_SOL_SOCKET, NN_RCVTIMEO, &to, sizeof (to));
+  string hello = "hello";
+  sock.send(hello.c_str(), hello.size()+1, 0);
+
+  while (true) {
+    char* buf = NULL;
+    size_t sz = sock.recv(&buf, NN_MSG, 0);
+    if (buf) {
+      const string in(buf, buf+sz);
+      nn::freemsg(buf);
+      if (in == "shutdown") {
+        string off = "off";
+        sock.send(off.c_str(), off.size()+1, 0);
+        break;
+      }
+      vector<string> parts;
+      boost::algorithm::split_regex(parts, in, boost::regex(" \\|\\|\\| "));
+      boost::shared_ptr<Array2D<bool> > out(new Array2D<bool>);
+      boost::shared_ptr<Array2D<bool> > a1 = AlignmentIO::ReadPharaohAlignmentGrid(parts[0]);
+      boost::shared_ptr<Array2D<bool> > a2 = AlignmentIO::ReadPharaohAlignmentGrid(parts[1]);
+      cmd.Apply(*a1, *a2, out.get());
+      ostringstream os;
+      if (cmd.Result() == 1) {
+        AlignmentIO::SerializePharaohFormat(*out, &os);
+      } else {
+        // error
+      }
+      sock.send(os.str().c_str(), os.str().size()+1, 0);
+    }
+  }
+
+  return 0;
+}
+
diff --git a/word-aligner/Makefile.am b/word-aligner/Makefile.am
index 071e4977..2bffa267 100644
--- a/word-aligner/Makefile.am
+++ b/word-aligner/Makefile.am
@@ -1,4 +1,4 @@
-bin_PROGRAMS = fast_align binderiv
+bin_PROGRAMS = fast_align binderiv net_fa
 
 fast_align_SOURCES = fast_align.cc ttables.cc da.h ttables.h
 fast_align_LDADD = ../utils/libutils.a
@@ -7,6 +7,11 @@ fast_align_LDFLAGS = $(STATIC_FLAGS)
 binderiv_SOURCES = binderiv.cc
 binderiv_LDADD = ../utils/libutils.a
 
+net_fa_SOURCES = net_fa.cc ttables.cc da.h ttables.h nn.hpp
+net_fa_LDADD = ../utils/libutils.a
+net_fa_LDFLAGS = $(STATIC_FLAGS) /srv/postedit/lib/nanomsg-0.5-beta/lib/libnanomsg.so
+
 EXTRA_DIST = aligner.pl ortho-norm support makefiles stemmers
 
-AM_CPPFLAGS = -W -Wall -I$(top_srcdir) -I$(top_srcdir)/utils -I$(top_srcdir)/training
+AM_CPPFLAGS = -W -Wall -I$(top_srcdir) -I$(top_srcdir)/utils -I$(top_srcdir)/training -I/srv/postedit/lib/nanomsg-0.5-beta/include -I/srv/postedit/lib/cppnanomsg
+
diff --git a/word-aligner/net_fa.cc b/word-aligner/net_fa.cc
new file mode 100644
index 00000000..f71c3f04
--- /dev/null
+++ b/word-aligner/net_fa.cc
@@ -0,0 +1,134 @@
+#include <iostream>
+#include <cmath>
+#include <utility>
+#ifndef HAVE_OLD_CPP
+# include <unordered_map>
+#else
+# include <tr1/unordered_map>
+namespace std { using std::tr1::unordered_map; }
+#endif
+
+#include <boost/functional/hash.hpp>
+#include <boost/program_options.hpp>
+#include <boost/program_options/variables_map.hpp>
+
+#include "m.h"
+#include "corpus_tools.h"
+#include "stringlib.h"
+#include "filelib.h"
+#include "ttables.h"
+#include "tdict.h"
+#include "da.h"
+
+#include <nanomsg/nn.h>
+#include <nanomsg/pair.h>
+#include "nn.hpp"
+
+namespace po = boost::program_options;
+using namespace std;
+
+bool InitCommandLine(int argc, char** argv, po::variables_map* conf) {
+  po::options_description opts("Configuration options");
+  opts.add_options()
+    ("diagonal_tension,T", po::value<double>()->default_value(4.0), "How sharp or flat around the diagonal is the alignment distribution (<1 = flat >1 = sharp)")
+		("mean_srclen_multiplier,m",po::value<double>()->default_value(1), "When --force_align, use this source length multiplier")
+		("force_align,f",po::value<string>(), "Load previously written parameters to 'force align' input. Set --diagonal_tension and --mean_srclen_multiplier as estimated during training.")
+    ("favor_diagonal,d", "Use a static alignment distribution that assigns higher probabilities to alignments near the diagonal")
+    ("prob_align_null", po::value<double>()->default_value(0.08), "When --favor_diagonal is set, what's the probability of a null alignment?")
+    ("no_null_word,N","Do not generate from a null token")
+    ("sock_url", po::value<string>()->default_value("tcp://127.0.0.1:60666"), "Socket url.");
+  po::options_description clo("Command line options");
+  clo.add_options()
+        ("help,h", "Print this help message and exit");
+  po::options_description dconfig_options, dcmdline_options;
+  dconfig_options.add(opts);
+  dcmdline_options.add(opts).add(clo);
+
+  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
+  po::notify(*conf);
+
+  if (conf->count("help") || conf->count("force_align")==0) {
+    cerr << "Usage " << argv[0] << " [OPTIONS] -f params\n";
+    cerr << dcmdline_options << endl;
+    return false;
+  }
+
+  return true;
+}
+
+int main(int argc, char** argv) {
+  po::variables_map conf;
+  if (!InitCommandLine(argc, argv, &conf)) return 1;
+  const double diagonal_tension = conf["diagonal_tension"].as<double>();
+	const double mean_srclen_multiplier = conf["mean_srclen_multiplier"].as<double>();
+  const bool use_null = (conf.count("no_null_word") == 0);
+  const bool favor_diagonal = conf.count("favor_diagonal");
+  const double prob_align_null = conf["prob_align_null"].as<double>();
+  const double prob_align_not_null = 1.0 - prob_align_null;
+  const WordID kNULL = TD::Convert("<eps>");
+	ReadFile s2t_f(conf["force_align"].as<string>());
+  TTable s2t;
+	s2t.DeserializeLogProbsFromText(s2t_f.stream());
+
+  nn::socket sock(AF_SP, NN_PAIR);
+  string url = conf["sock_url"].as<string>();
+  sock.bind(url.c_str());
+  int to = 100;
+  sock.setsockopt(NN_SOL_SOCKET, NN_RCVTIMEO, &to, sizeof (to));
+  string hello = "hello";
+  sock.send(hello.c_str(), hello.size()+1, 0);
+
+  while (true)
+  {
+    char *buf = NULL;
+    size_t sz = sock.recv(&buf, NN_MSG, 0);
+    if (!buf)
+      continue;
+    string line(buf, buf+sz);
+    if (line == "shutdown") {
+      cerr << "[net_fa] shutting down" << endl;
+      string shutdown = "off";
+      sock.send(shutdown.c_str(), shutdown.size()+1, 0);
+      break;
+    }
+    cerr << "[net_fa] got '" << line << "'" << endl;
+    nn::freemsg(buf);
+    vector<WordID> src, trg;
+    CorpusTools::ReadLine(line, &src, &trg);
+    double log_prob = Md::log_poisson(trg.size(), 0.05 + src.size() * mean_srclen_multiplier);
+
+    // compute likelihood
+    ostringstream ss;
+    for (unsigned j = 0; j < trg.size(); ++j) {
+      const WordID& f_j = trg[j];
+      double sum = 0;
+      int a_j = 0;
+      double max_pat = 0;
+      double prob_a_i = 1.0 / (src.size() + use_null);  // uniform (model 1)
+      if (use_null) {
+        if (favor_diagonal) prob_a_i = prob_align_null;
+        max_pat = s2t.prob(kNULL, f_j) * prob_a_i;
+        sum += max_pat;
+      }
+      double az = 0;
+      if (favor_diagonal)
+        az = DiagonalAlignment::ComputeZ(j+1, trg.size(), src.size(), diagonal_tension) / prob_align_not_null;
+      for (unsigned i = 1; i <= src.size(); ++i) {
+        if (favor_diagonal)
+          prob_a_i = DiagonalAlignment::UnnormalizedProb(j + 1, i, trg.size(), src.size(), diagonal_tension) / az;
+        double pat = s2t.prob(src[i-1], f_j) * prob_a_i;
+        if (pat > max_pat) { max_pat = pat; a_j = i; }
+        sum += pat;
+      }
+      log_prob += log(sum);
+      if (a_j > 0)
+        ss << ' ' << (a_j - 1) << '-' << j;
+    }
+    string a = ss.str();
+    cerr << "[net_fa] sending '" << a << "'" << endl;
+    sock.send(a.c_str(), a.size()+1, 0);
+  } // loop
+
+  return 0;
+}
+