initial checkin

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@2 ec762483-ff6d-05da-a07a-a48fb63a330f

110 files changed, 17207 insertions, 0 deletions

diff --git a/decoder/JSON_parser.c b/decoder/JSON_parser.c
new file mode 100644
index 00000000..175b7cc9
--- /dev/null
+++ b/decoder/JSON_parser.c
@@ -0,0 +1,1012 @@
+/* JSON_parser.c */
+
+/* 2007-08-24 */
+
+/*
+Copyright (c) 2005 JSON.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+The Software shall be used for Good, not Evil.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+/*
+    Callbacks, comments, Unicode handling by Jean Gressmann (jean@0x42.de), 2007-2009.
+    
+    For the added features the license above applies also.
+    
+    Changelog:
+        2009-05-17 
+            Incorporated benrudiak@googlemail.com fix for UTF16 decoding.
+            
+        2009-05-14 
+            Fixed float parsing bug related to a locale being set that didn't
+            use '.' as decimal point character (charles@transmissionbt.com).
+            
+        2008-10-14 
+            Renamed states.IN to states.IT to avoid name clash which IN macro
+            defined in windef.h (alexey.pelykh@gmail.com)
+            
+        2008-07-19 
+            Removed some duplicate code & debugging variable (charles@transmissionbt.com)
+        
+        2008-05-28 
+            Made JSON_value structure ansi C compliant. This bug was report by 
+            trisk@acm.jhu.edu
+        
+        2008-05-20 
+            Fixed bug reported by charles@transmissionbt.com where the switching 
+            from static to dynamic parse buffer did not copy the static parse 
+            buffer's content.
+*/
+
+
+
+#include <assert.h>
+#include <ctype.h>
+#include <float.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <locale.h>
+
+#include "JSON_parser.h"
+
+#ifdef _MSC_VER
+#   if _MSC_VER >= 1400 /* Visual Studio 2005 and up */
+#      pragma warning(disable:4996) // unsecure sscanf
+#   endif
+#endif
+
+
+#define true  1
+#define false 0
+#define __   -1     /* the universal error code */
+
+/* values chosen so that the object size is approx equal to one page (4K) */
+#ifndef JSON_PARSER_STACK_SIZE
+#   define JSON_PARSER_STACK_SIZE 128
+#endif
+
+#ifndef JSON_PARSER_PARSE_BUFFER_SIZE
+#   define JSON_PARSER_PARSE_BUFFER_SIZE 3500
+#endif
+
+typedef unsigned short UTF16;
+
+struct JSON_parser_struct {
+    JSON_parser_callback callback;
+    void* ctx;
+    signed char state, before_comment_state, type, escaped, comment, allow_comments, handle_floats_manually;
+    UTF16 utf16_high_surrogate;
+    long depth;
+    long top;
+    signed char* stack;
+    long stack_capacity;
+    char decimal_point;
+    char* parse_buffer;
+    size_t parse_buffer_capacity;
+    size_t parse_buffer_count;
+    size_t comment_begin_offset;
+    signed char static_stack[JSON_PARSER_STACK_SIZE];
+    char static_parse_buffer[JSON_PARSER_PARSE_BUFFER_SIZE];
+};
+
+#define COUNTOF(x) (sizeof(x)/sizeof(x[0])) 
+
+/*
+    Characters are mapped into these character classes. This allows for
+    a significant reduction in the size of the state transition table.
+*/
+
+
+
+enum classes {
+    C_SPACE,  /* space */
+    C_WHITE,  /* other whitespace */
+    C_LCURB,  /* {  */
+    C_RCURB,  /* } */
+    C_LSQRB,  /* [ */
+    C_RSQRB,  /* ] */
+    C_COLON,  /* : */
+    C_COMMA,  /* , */
+    C_QUOTE,  /* " */
+    C_BACKS,  /* \ */
+    C_SLASH,  /* / */
+    C_PLUS,   /* + */
+    C_MINUS,  /* - */
+    C_POINT,  /* . */
+    C_ZERO ,  /* 0 */
+    C_DIGIT,  /* 123456789 */
+    C_LOW_A,  /* a */
+    C_LOW_B,  /* b */
+    C_LOW_C,  /* c */
+    C_LOW_D,  /* d */
+    C_LOW_E,  /* e */
+    C_LOW_F,  /* f */
+    C_LOW_L,  /* l */
+    C_LOW_N,  /* n */
+    C_LOW_R,  /* r */
+    C_LOW_S,  /* s */
+    C_LOW_T,  /* t */
+    C_LOW_U,  /* u */
+    C_ABCDF,  /* ABCDF */
+    C_E,      /* E */
+    C_ETC,    /* everything else */
+    C_STAR,   /* * */   
+    NR_CLASSES
+};
+
+static int ascii_class[128] = {
+/*
+    This array maps the 128 ASCII characters into character classes.
+    The remaining Unicode characters should be mapped to C_ETC.
+    Non-whitespace control characters are errors.
+*/
+    __,      __,      __,      __,      __,      __,      __,      __,
+    __,      C_WHITE, C_WHITE, __,      __,      C_WHITE, __,      __,
+    __,      __,      __,      __,      __,      __,      __,      __,
+    __,      __,      __,      __,      __,      __,      __,      __,
+
+    C_SPACE, C_ETC,   C_QUOTE, C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,
+    C_ETC,   C_ETC,   C_STAR,   C_PLUS,  C_COMMA, C_MINUS, C_POINT, C_SLASH,
+    C_ZERO,  C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT, C_DIGIT,
+    C_DIGIT, C_DIGIT, C_COLON, C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,
+
+    C_ETC,   C_ABCDF, C_ABCDF, C_ABCDF, C_ABCDF, C_E,     C_ABCDF, C_ETC,
+    C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,
+    C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_ETC,
+    C_ETC,   C_ETC,   C_ETC,   C_LSQRB, C_BACKS, C_RSQRB, C_ETC,   C_ETC,
+
+    C_ETC,   C_LOW_A, C_LOW_B, C_LOW_C, C_LOW_D, C_LOW_E, C_LOW_F, C_ETC,
+    C_ETC,   C_ETC,   C_ETC,   C_ETC,   C_LOW_L, C_ETC,   C_LOW_N, C_ETC,
+    C_ETC,   C_ETC,   C_LOW_R, C_LOW_S, C_LOW_T, C_LOW_U, C_ETC,   C_ETC,
+    C_ETC,   C_ETC,   C_ETC,   C_LCURB, C_ETC,   C_RCURB, C_ETC,   C_ETC
+};
+
+
+/*
+    The state codes.
+*/
+enum states {
+    GO,  /* start    */
+    OK,  /* ok       */
+    OB,  /* object   */
+    KE,  /* key      */
+    CO,  /* colon    */
+    VA,  /* value    */
+    AR,  /* array    */
+    ST,  /* string   */
+    ES,  /* escape   */
+    U1,  /* u1       */
+    U2,  /* u2       */
+    U3,  /* u3       */
+    U4,  /* u4       */
+    MI,  /* minus    */
+    ZE,  /* zero     */
+    IT,  /* integer  */
+    FR,  /* fraction */
+    E1,  /* e        */
+    E2,  /* ex       */
+    E3,  /* exp      */
+    T1,  /* tr       */
+    T2,  /* tru      */
+    T3,  /* true     */
+    F1,  /* fa       */
+    F2,  /* fal      */
+    F3,  /* fals     */
+    F4,  /* false    */
+    N1,  /* nu       */
+    N2,  /* nul      */
+    N3,  /* null     */
+    C1,  /* /        */
+    C2,  /* / *     */
+    C3,  /* *        */
+    FX,  /* *.* *eE* */
+    D1,  /* second UTF-16 character decoding started by \ */
+    D2,  /* second UTF-16 character proceeded by u */
+    NR_STATES
+};
+
+enum actions
+{
+    CB = -10, /* comment begin */
+    CE = -11, /* comment end */
+    FA = -12, /* false */
+    TR = -13, /* false */
+    NU = -14, /* null */
+    DE = -15, /* double detected by exponent e E */
+    DF = -16, /* double detected by fraction . */
+    SB = -17, /* string begin */
+    MX = -18, /* integer detected by minus */
+    ZX = -19, /* integer detected by zero */
+    IX = -20, /* integer detected by 1-9 */
+    EX = -21, /* next char is escaped */
+    UC = -22  /* Unicode character read */
+};
+
+
+static int state_transition_table[NR_STATES][NR_CLASSES] = {
+/*
+    The state transition table takes the current state and the current symbol,
+    and returns either a new state or an action. An action is represented as a
+    negative number. A JSON text is accepted if at the end of the text the
+    state is OK and if the mode is MODE_DONE.
+
+                 white                                      1-9                                   ABCDF  etc
+             space |  {  }  [  ]  :  ,  "  \  /  +  -  .  0  |  a  b  c  d  e  f  l  n  r  s  t  u  |  E  |  * */
+/*start  GO*/ {GO,GO,-6,__,-5,__,__,__,__,__,CB,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*ok     OK*/ {OK,OK,__,-8,__,-7,__,-3,__,__,CB,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*object OB*/ {OB,OB,__,-9,__,__,__,__,SB,__,CB,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*key    KE*/ {KE,KE,__,__,__,__,__,__,SB,__,CB,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*colon  CO*/ {CO,CO,__,__,__,__,-2,__,__,__,CB,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*value  VA*/ {VA,VA,-6,__,-5,__,__,__,SB,__,CB,__,MX,__,ZX,IX,__,__,__,__,__,FA,__,NU,__,__,TR,__,__,__,__,__},
+/*array  AR*/ {AR,AR,-6,__,-5,-7,__,__,SB,__,CB,__,MX,__,ZX,IX,__,__,__,__,__,FA,__,NU,__,__,TR,__,__,__,__,__},
+/*string ST*/ {ST,__,ST,ST,ST,ST,ST,ST,-4,EX,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST,ST},
+/*escape ES*/ {__,__,__,__,__,__,__,__,ST,ST,ST,__,__,__,__,__,__,ST,__,__,__,ST,__,ST,ST,__,ST,U1,__,__,__,__},
+/*u1     U1*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,U2,U2,U2,U2,U2,U2,U2,U2,__,__,__,__,__,__,U2,U2,__,__},
+/*u2     U2*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,U3,U3,U3,U3,U3,U3,U3,U3,__,__,__,__,__,__,U3,U3,__,__},
+/*u3     U3*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,U4,U4,U4,U4,U4,U4,U4,U4,__,__,__,__,__,__,U4,U4,__,__},
+/*u4     U4*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,UC,UC,UC,UC,UC,UC,UC,UC,__,__,__,__,__,__,UC,UC,__,__},
+/*minus  MI*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,ZE,IT,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*zero   ZE*/ {OK,OK,__,-8,__,-7,__,-3,__,__,CB,__,__,DF,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*int    IT*/ {OK,OK,__,-8,__,-7,__,-3,__,__,CB,__,__,DF,IT,IT,__,__,__,__,DE,__,__,__,__,__,__,__,__,DE,__,__},
+/*frac   FR*/ {OK,OK,__,-8,__,-7,__,-3,__,__,CB,__,__,__,FR,FR,__,__,__,__,E1,__,__,__,__,__,__,__,__,E1,__,__},
+/*e      E1*/ {__,__,__,__,__,__,__,__,__,__,__,E2,E2,__,E3,E3,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*ex     E2*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,E3,E3,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*exp    E3*/ {OK,OK,__,-8,__,-7,__,-3,__,__,__,__,__,__,E3,E3,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*tr     T1*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,T2,__,__,__,__,__,__,__},
+/*tru    T2*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,T3,__,__,__,__},
+/*true   T3*/ {__,__,__,__,__,__,__,__,__,__,CB,__,__,__,__,__,__,__,__,__,OK,__,__,__,__,__,__,__,__,__,__,__},
+/*fa     F1*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,F2,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*fal    F2*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,F3,__,__,__,__,__,__,__,__,__},
+/*fals   F3*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,F4,__,__,__,__,__,__},
+/*false  F4*/ {__,__,__,__,__,__,__,__,__,__,CB,__,__,__,__,__,__,__,__,__,OK,__,__,__,__,__,__,__,__,__,__,__},
+/*nu     N1*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,N2,__,__,__,__},
+/*nul    N2*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,N3,__,__,__,__,__,__,__,__,__},
+/*null   N3*/ {__,__,__,__,__,__,__,__,__,__,CB,__,__,__,__,__,__,__,__,__,__,__,OK,__,__,__,__,__,__,__,__,__},
+/*/      C1*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,C2},
+/*/*     C2*/ {C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C3},
+/**      C3*/ {C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,CE,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C2,C3},
+/*_.     FX*/ {OK,OK,__,-8,__,-7,__,-3,__,__,__,__,__,__,FR,FR,__,__,__,__,E1,__,__,__,__,__,__,__,__,E1,__,__},
+/*\      D1*/ {__,__,__,__,__,__,__,__,__,D2,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__},
+/*\      D2*/ {__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,U1,__,__,__,__},
+};
+
+
+/*
+    These modes can be pushed on the stack.
+*/
+enum modes {
+    MODE_ARRAY = 1, 
+    MODE_DONE = 2,  
+    MODE_KEY = 3,   
+    MODE_OBJECT = 4
+};
+
+static int
+push(JSON_parser jc, int mode)
+{
+/*
+    Push a mode onto the stack. Return false if there is overflow.
+*/
+    jc->top += 1;
+    if (jc->depth < 0) {
+        if (jc->top >= jc->stack_capacity) {
+            size_t bytes_to_allocate;
+            jc->stack_capacity *= 2;
+            bytes_to_allocate = jc->stack_capacity * sizeof(jc->static_stack[0]);
+            if (jc->stack == &jc->static_stack[0]) {
+                jc->stack = (signed char*)malloc(bytes_to_allocate);
+                memcpy(jc->stack, jc->static_stack, sizeof(jc->static_stack));
+            } else {
+                jc->stack = (signed char*)realloc(jc->stack, bytes_to_allocate);
+            }
+        }
+    } else {
+        if (jc->top >= jc->depth) {
+            return false;
+        }
+    }
+    
+    jc->stack[jc->top] = mode;
+    return true;
+}
+
+
+static int
+pop(JSON_parser jc, int mode)
+{
+/*
+    Pop the stack, assuring that the current mode matches the expectation.
+    Return false if there is underflow or if the modes mismatch.
+*/
+    if (jc->top < 0 || jc->stack[jc->top] != mode) {
+        return false;
+    }
+    jc->top -= 1;
+    return true;
+}
+
+
+#define parse_buffer_clear(jc) \
+    do {\
+        jc->parse_buffer_count = 0;\
+        jc->parse_buffer[0] = 0;\
+    } while (0)
+    
+#define parse_buffer_pop_back_char(jc)\
+    do {\
+        assert(jc->parse_buffer_count >= 1);\
+        --jc->parse_buffer_count;\
+        jc->parse_buffer[jc->parse_buffer_count] = 0;\
+    } while (0)    
+    
+void delete_JSON_parser(JSON_parser jc)
+{
+    if (jc) {
+        if (jc->stack != &jc->static_stack[0]) {
+            free((void*)jc->stack);
+        }
+        if (jc->parse_buffer != &jc->static_parse_buffer[0]) {
+            free((void*)jc->parse_buffer);
+        }
+        free((void*)jc);
+     }   
+}
+
+
+JSON_parser
+new_JSON_parser(JSON_config* config)
+{
+/*
+    new_JSON_parser starts the checking process by constructing a JSON_parser
+    object. It takes a depth parameter that restricts the level of maximum
+    nesting.
+
+    To continue the process, call JSON_parser_char for each character in the
+    JSON text, and then call JSON_parser_done to obtain the final result.
+    These functions are fully reentrant.
+*/
+
+    int depth = 0;
+    JSON_config default_config;
+    
+    JSON_parser jc = (JSON_parser)malloc(sizeof(struct JSON_parser_struct));
+    
+    memset(jc, 0, sizeof(*jc));
+    
+    
+    /* initialize configuration */
+    init_JSON_config(&default_config);
+    
+    /* set to default configuration if none was provided */
+    if (config == NULL) {
+        config = &default_config;
+    }
+
+    depth = config->depth;
+    
+    /* We need to be able to push at least one object */
+    if (depth == 0) {
+        depth = 1;
+    }
+    
+    jc->state = GO;
+    jc->top = -1;
+    
+    /* Do we want non-bound stack? */
+    if (depth > 0) {
+        jc->stack_capacity = depth;
+        jc->depth = depth;
+        if (depth <= (int)COUNTOF(jc->static_stack)) {
+            jc->stack = &jc->static_stack[0];
+        } else {
+            jc->stack = (signed char*)malloc(jc->stack_capacity * sizeof(jc->static_stack[0]));
+        }
+    } else {
+        jc->stack_capacity = COUNTOF(jc->static_stack);
+        jc->depth = -1;
+        jc->stack = &jc->static_stack[0];
+    }
+    
+    /* set parser to start */
+    push(jc, MODE_DONE);
+    
+    /* set up the parse buffer */
+    jc->parse_buffer = &jc->static_parse_buffer[0];
+    jc->parse_buffer_capacity = COUNTOF(jc->static_parse_buffer);
+    parse_buffer_clear(jc);
+    
+    /* set up callback, comment & float handling */
+    jc->callback = config->callback;
+    jc->ctx = config->callback_ctx;
+    jc->allow_comments = config->allow_comments != 0;
+    jc->handle_floats_manually = config->handle_floats_manually != 0;
+    
+    /* set up decimal point */
+    jc->decimal_point = *localeconv()->decimal_point;
+    
+    return jc;
+}
+
+static void grow_parse_buffer(JSON_parser jc)
+{
+    size_t bytes_to_allocate;
+    jc->parse_buffer_capacity *= 2;
+    bytes_to_allocate = jc->parse_buffer_capacity * sizeof(jc->parse_buffer[0]);
+    if (jc->parse_buffer == &jc->static_parse_buffer[0]) {
+        jc->parse_buffer = (char*)malloc(bytes_to_allocate);
+        memcpy(jc->parse_buffer, jc->static_parse_buffer, jc->parse_buffer_count);
+    } else {
+        jc->parse_buffer = (char*)realloc(jc->parse_buffer, bytes_to_allocate);
+    }
+}
+
+#define parse_buffer_push_back_char(jc, c)\
+    do {\
+        if (jc->parse_buffer_count + 1 >= jc->parse_buffer_capacity) grow_parse_buffer(jc);\
+        jc->parse_buffer[jc->parse_buffer_count++] = c;\
+        jc->parse_buffer[jc->parse_buffer_count]   = 0;\
+    } while (0)
+
+#define assert_is_non_container_type(jc) \
+    assert( \
+        jc->type == JSON_T_NULL || \
+        jc->type == JSON_T_FALSE || \
+        jc->type == JSON_T_TRUE || \
+        jc->type == JSON_T_FLOAT || \
+        jc->type == JSON_T_INTEGER || \
+        jc->type == JSON_T_STRING)
+    
+
+static int parse_parse_buffer(JSON_parser jc)
+{
+    if (jc->callback) {
+        JSON_value value, *arg = NULL;
+        
+        if (jc->type != JSON_T_NONE) {
+            assert_is_non_container_type(jc);
+        
+            switch(jc->type) {
+                case JSON_T_FLOAT:
+                    arg = &value;
+                    if (jc->handle_floats_manually) {
+                        value.vu.str.value = jc->parse_buffer;
+                        value.vu.str.length = jc->parse_buffer_count;
+                    } else { 
+                        /*sscanf(jc->parse_buffer, "%Lf", &value.vu.float_value);*/
+                        
+                        /* not checking with end pointer b/c there may be trailing ws */
+                        value.vu.float_value = strtold(jc->parse_buffer, NULL);
+                    }
+                    break;
+                case JSON_T_INTEGER:
+                    arg = &value;
+                    sscanf(jc->parse_buffer, JSON_PARSER_INTEGER_SSCANF_TOKEN, &value.vu.integer_value);
+                    break;
+                case JSON_T_STRING:
+                    arg = &value;
+                    value.vu.str.value = jc->parse_buffer;
+                    value.vu.str.length = jc->parse_buffer_count;
+                    break;
+            }
+            
+            if (!(*jc->callback)(jc->ctx, jc->type, arg)) {
+                return false;
+            }
+        }
+    }
+    
+    parse_buffer_clear(jc);
+    
+    return true;
+}
+
+#define IS_HIGH_SURROGATE(uc) (((uc) & 0xFC00) == 0xD800)
+#define IS_LOW_SURROGATE(uc)  (((uc) & 0xFC00) == 0xDC00)
+#define DECODE_SURROGATE_PAIR(hi,lo) ((((hi) & 0x3FF) << 10) + ((lo) & 0x3FF) + 0x10000)
+static unsigned char utf8_lead_bits[4] = { 0x00, 0xC0, 0xE0, 0xF0 };
+
+static int decode_unicode_char(JSON_parser jc)
+{
+    int i;
+    unsigned uc = 0;
+    char* p;
+    int trail_bytes;
+    
+    assert(jc->parse_buffer_count >= 6);
+    
+    p = &jc->parse_buffer[jc->parse_buffer_count - 4];
+    
+    for (i = 12; i >= 0; i -= 4, ++p) {
+        unsigned x = *p;
+        
+        if (x >= 'a') {
+            x -= ('a' - 10);
+        } else if (x >= 'A') {
+            x -= ('A' - 10);
+        } else {
+            x &= ~0x30u;
+        }
+        
+        assert(x < 16);
+        
+        uc |= x << i;
+    }
+    
+    /* clear UTF-16 char from buffer */
+    jc->parse_buffer_count -= 6;
+    jc->parse_buffer[jc->parse_buffer_count] = 0;
+    
+    /* attempt decoding ... */
+    if (jc->utf16_high_surrogate) {
+        if (IS_LOW_SURROGATE(uc)) {
+            uc = DECODE_SURROGATE_PAIR(jc->utf16_high_surrogate, uc);
+            trail_bytes = 3;
+            jc->utf16_high_surrogate = 0;
+        } else {
+            /* high surrogate without a following low surrogate */
+            return false;
+        }
+    } else {
+        if (uc < 0x80) {
+            trail_bytes = 0;
+        } else if (uc < 0x800) {
+            trail_bytes = 1;
+        } else if (IS_HIGH_SURROGATE(uc)) {
+            /* save the high surrogate and wait for the low surrogate */
+            jc->utf16_high_surrogate = uc;
+            return true;
+        } else if (IS_LOW_SURROGATE(uc)) {
+            /* low surrogate without a preceding high surrogate */
+            return false;
+        } else {
+            trail_bytes = 2;
+        }
+    }
+    
+    jc->parse_buffer[jc->parse_buffer_count++] = (char) ((uc >> (trail_bytes * 6)) | utf8_lead_bits[trail_bytes]);
+    
+    for (i = trail_bytes * 6 - 6; i >= 0; i -= 6) {
+        jc->parse_buffer[jc->parse_buffer_count++] = (char) (((uc >> i) & 0x3F) | 0x80);
+    }
+
+    jc->parse_buffer[jc->parse_buffer_count] = 0;
+    
+    return true;
+}
+
+static int add_escaped_char_to_parse_buffer(JSON_parser jc, int next_char)
+{
+    jc->escaped = 0;
+    /* remove the backslash */
+    parse_buffer_pop_back_char(jc);
+    switch(next_char) {
+        case 'b':
+            parse_buffer_push_back_char(jc, '\b');
+            break;
+        case 'f':
+            parse_buffer_push_back_char(jc, '\f');
+            break;
+        case 'n':
+            parse_buffer_push_back_char(jc, '\n');
+            break;
+        case 'r':
+            parse_buffer_push_back_char(jc, '\r');
+            break;
+        case 't':
+            parse_buffer_push_back_char(jc, '\t');
+            break;
+        case '"':
+            parse_buffer_push_back_char(jc, '"');
+            break;
+        case '\\':
+            parse_buffer_push_back_char(jc, '\\');
+            break;
+        case '/':
+            parse_buffer_push_back_char(jc, '/');
+            break;
+        case 'u':
+            parse_buffer_push_back_char(jc, '\\');
+            parse_buffer_push_back_char(jc, 'u');
+            break;
+        default:
+            return false;
+    }
+
+    return true;
+}
+
+#define add_char_to_parse_buffer(jc, next_char, next_class) \
+    do { \
+        if (jc->escaped) { \
+            if (!add_escaped_char_to_parse_buffer(jc, next_char)) \
+                return false; \
+        } else if (!jc->comment) { \
+            if ((jc->type != JSON_T_NONE) | !((next_class == C_SPACE) | (next_class == C_WHITE)) /* non-white-space */) { \
+                parse_buffer_push_back_char(jc, (char)next_char); \
+            } \
+        } \
+    } while (0)
+    
+
+#define assert_type_isnt_string_null_or_bool(jc) \
+    assert(jc->type != JSON_T_FALSE); \
+    assert(jc->type != JSON_T_TRUE); \
+    assert(jc->type != JSON_T_NULL); \
+    assert(jc->type != JSON_T_STRING)
+
+
+int
+JSON_parser_char(JSON_parser jc, int next_char)
+{
+/*
+    After calling new_JSON_parser, call this function for each character (or
+    partial character) in your JSON text. It can accept UTF-8, UTF-16, or
+    UTF-32. It returns true if things are looking ok so far. If it rejects the
+    text, it returns false.
+*/
+    int next_class, next_state;
+    
+/*
+    Determine the character's class.
+*/
+    if (next_char < 0) {
+        return false;
+    }
+    if (next_char >= 128) {
+        next_class = C_ETC;
+    } else {
+        next_class = ascii_class[next_char];
+        if (next_class <= __) {
+            return false;
+        }
+    }
+    
+    add_char_to_parse_buffer(jc, next_char, next_class);
+    
+/*
+    Get the next state from the state transition table.
+*/
+    next_state = state_transition_table[jc->state][next_class];
+    if (next_state >= 0) {
+/*
+    Change the state.
+*/
+        jc->state = next_state;
+    } else {
+/*
+    Or perform one of the actions.
+*/
+        switch (next_state) {
+/* Unicode character */        
+        case UC:
+            if(!decode_unicode_char(jc)) {
+                return false;
+            }
+            /* check if we need to read a second UTF-16 char */
+            if (jc->utf16_high_surrogate) {
+                jc->state = D1;
+            } else {
+                jc->state = ST;
+            }
+            break;
+/* escaped char */
+        case EX:
+            jc->escaped = 1;
+            jc->state = ES;
+            break;
+/* integer detected by minus */
+        case MX:
+            jc->type = JSON_T_INTEGER;
+            jc->state = MI;
+            break;  
+/* integer detected by zero */            
+        case ZX:
+            jc->type = JSON_T_INTEGER;
+            jc->state = ZE;
+            break;  
+/* integer detected by 1-9 */            
+        case IX:
+            jc->type = JSON_T_INTEGER;
+            jc->state = IT;
+            break;  
+            
+/* floating point number detected by exponent*/
+        case DE:
+            assert_type_isnt_string_null_or_bool(jc);
+            jc->type = JSON_T_FLOAT;
+            jc->state = E1;
+            break;   
+        
+/* floating point number detected by fraction */
+        case DF:
+            assert_type_isnt_string_null_or_bool(jc);
+            if (!jc->handle_floats_manually) {
+/*
+    Some versions of strtod (which underlies sscanf) don't support converting 
+    C-locale formated floating point values.
+*/           
+                assert(jc->parse_buffer[jc->parse_buffer_count-1] == '.');
+                jc->parse_buffer[jc->parse_buffer_count-1] = jc->decimal_point;
+            }            
+            jc->type = JSON_T_FLOAT;
+            jc->state = FX;
+            break;   
+/* string begin " */
+        case SB:
+            parse_buffer_clear(jc);
+            assert(jc->type == JSON_T_NONE);
+            jc->type = JSON_T_STRING;
+            jc->state = ST;
+            break;        
+        
+/* n */
+        case NU:
+            assert(jc->type == JSON_T_NONE);
+            jc->type = JSON_T_NULL;
+            jc->state = N1;
+            break;        
+/* f */
+        case FA:
+            assert(jc->type == JSON_T_NONE);
+            jc->type = JSON_T_FALSE;
+            jc->state = F1;
+            break;        
+/* t */
+        case TR:
+            assert(jc->type == JSON_T_NONE);
+            jc->type = JSON_T_TRUE;
+            jc->state = T1;
+            break;        
+        
+/* closing comment */
+        case CE:
+            jc->comment = 0;
+            assert(jc->parse_buffer_count == 0);
+            assert(jc->type == JSON_T_NONE);
+            jc->state = jc->before_comment_state;
+            break;        
+        
+/* opening comment  */
+        case CB:
+            if (!jc->allow_comments) {
+                return false;
+            }
+            parse_buffer_pop_back_char(jc);
+            if (!parse_parse_buffer(jc)) {
+                return false;
+            }
+            assert(jc->parse_buffer_count == 0);
+            assert(jc->type != JSON_T_STRING);
+            switch (jc->stack[jc->top]) {
+            case MODE_ARRAY:
+            case MODE_OBJECT:   
+                switch(jc->state) {
+                case VA:
+                case AR:
+                    jc->before_comment_state = jc->state;
+                    break;
+                default:
+                    jc->before_comment_state = OK;
+                    break;
+                }
+                break;
+            default:
+                jc->before_comment_state = jc->state;
+                break;
+            }
+            jc->type = JSON_T_NONE;
+            jc->state = C1;
+            jc->comment = 1;
+            break;
+/* empty } */
+        case -9:        
+            parse_buffer_clear(jc);
+            if (jc->callback && !(*jc->callback)(jc->ctx, JSON_T_OBJECT_END, NULL)) {
+                return false;
+            }
+            if (!pop(jc, MODE_KEY)) {
+                return false;
+            }
+            jc->state = OK;
+            break;
+
+/* } */ case -8:
+            parse_buffer_pop_back_char(jc);
+            if (!parse_parse_buffer(jc)) {
+                return false;
+            }
+            if (jc->callback && !(*jc->callback)(jc->ctx, JSON_T_OBJECT_END, NULL)) {
+                return false;
+            }
+            if (!pop(jc, MODE_OBJECT)) {
+                return false;
+            }
+            jc->type = JSON_T_NONE;
+            jc->state = OK;
+            break;
+
+/* ] */ case -7:
+            parse_buffer_pop_back_char(jc);
+            if (!parse_parse_buffer(jc)) {
+                return false;
+            }
+            if (jc->callback && !(*jc->callback)(jc->ctx, JSON_T_ARRAY_END, NULL)) {
+                return false;
+            }
+            if (!pop(jc, MODE_ARRAY)) {
+                return false;
+            }
+            
+            jc->type = JSON_T_NONE;
+            jc->state = OK;
+            break;
+
+/* { */ case -6:
+            parse_buffer_pop_back_char(jc);
+            if (jc->callback && !(*jc->callback)(jc->ctx, JSON_T_OBJECT_BEGIN, NULL)) {
+                return false;
+            }
+            if (!push(jc, MODE_KEY)) {
+                return false;
+            }
+            assert(jc->type == JSON_T_NONE);
+            jc->state = OB;
+            break;
+
+/* [ */ case -5:
+            parse_buffer_pop_back_char(jc);
+            if (jc->callback && !(*jc->callback)(jc->ctx, JSON_T_ARRAY_BEGIN, NULL)) {
+                return false;
+            }
+            if (!push(jc, MODE_ARRAY)) {
+                return false;
+            }
+            assert(jc->type == JSON_T_NONE);
+            jc->state = AR;
+            break;
+
+/* string end " */ case -4:
+            parse_buffer_pop_back_char(jc);
+            switch (jc->stack[jc->top]) {
+            case MODE_KEY:
+                assert(jc->type == JSON_T_STRING);
+                jc->type = JSON_T_NONE;
+                jc->state = CO;
+                
+                if (jc->callback) {
+                    JSON_value value;
+                    value.vu.str.value = jc->parse_buffer;
+                    value.vu.str.length = jc->parse_buffer_count;
+                    if (!(*jc->callback)(jc->ctx, JSON_T_KEY, &value)) {
+                        return false;
+                    }
+                }
+                parse_buffer_clear(jc);
+                break;
+            case MODE_ARRAY:
+            case MODE_OBJECT:
+                assert(jc->type == JSON_T_STRING);
+                if (!parse_parse_buffer(jc)) {
+                    return false;
+                }
+                jc->type = JSON_T_NONE;
+                jc->state = OK;
+                break;
+            default:
+                return false;
+            }
+            break;
+
+/* , */ case -3:
+            parse_buffer_pop_back_char(jc);
+            if (!parse_parse_buffer(jc)) {
+                return false;
+            }
+            switch (jc->stack[jc->top]) {
+            case MODE_OBJECT:
+/*
+    A comma causes a flip from object mode to key mode.
+*/
+                if (!pop(jc, MODE_OBJECT) || !push(jc, MODE_KEY)) {
+                    return false;
+                }
+                assert(jc->type != JSON_T_STRING);
+                jc->type = JSON_T_NONE;
+                jc->state = KE;
+                break;
+            case MODE_ARRAY:
+                assert(jc->type != JSON_T_STRING);
+                jc->type = JSON_T_NONE;
+                jc->state = VA;
+                break;
+            default:
+                return false;
+            }
+            break;
+
+/* : */ case -2:
+/*
+    A colon causes a flip from key mode to object mode.
+*/
+            parse_buffer_pop_back_char(jc);
+            if (!pop(jc, MODE_KEY) || !push(jc, MODE_OBJECT)) {
+                return false;
+            }
+            assert(jc->type == JSON_T_NONE);
+            jc->state = VA;
+            break;
+/*
+    Bad action.
+*/
+        default:
+            return false;
+        }
+    }
+    return true;
+}
+
+
+int
+JSON_parser_done(JSON_parser jc)
+{
+    const int result = jc->state == OK && pop(jc, MODE_DONE);
+
+    return result;
+}
+
+
+int JSON_parser_is_legal_white_space_string(const char* s)
+{
+    int c, char_class;
+    
+    if (s == NULL) {
+        return false;
+    }
+    
+    for (; *s; ++s) {   
+        c = *s;
+        
+        if (c < 0 || c >= 128) {
+            return false;
+        }
+        
+        char_class = ascii_class[c];
+        
+        if (char_class != C_SPACE && char_class != C_WHITE) {
+            return false;
+        }
+    }
+    
+    return true;
+}
+
+
+
+void init_JSON_config(JSON_config* config)
+{
+    if (config) {
+        memset(config, 0, sizeof(*config));
+        
+        config->depth = JSON_PARSER_STACK_SIZE - 1;
+    }
+}
diff --git a/decoder/JSON_parser.h b/decoder/JSON_parser.h
new file mode 100644
index 00000000..ceb5b24b
--- /dev/null
+++ b/decoder/JSON_parser.h
@@ -0,0 +1,152 @@
+#ifndef JSON_PARSER_H
+#define JSON_PARSER_H
+
+/* JSON_parser.h */
+
+
+#include <stddef.h>
+
+/* Windows DLL stuff */
+#ifdef _WIN32
+#	ifdef JSON_PARSER_DLL_EXPORTS
+#		define JSON_PARSER_DLL_API __declspec(dllexport)
+#	else
+#		define JSON_PARSER_DLL_API __declspec(dllimport)
+#   endif
+#else
+#	define JSON_PARSER_DLL_API 
+#endif
+
+/* Determine the integer type use to parse non-floating point numbers */
+#if __STDC_VERSION__ >= 199901L || HAVE_LONG_LONG == 1
+typedef long long JSON_int_t;
+#define JSON_PARSER_INTEGER_SSCANF_TOKEN "%lld"
+#define JSON_PARSER_INTEGER_SPRINTF_TOKEN "%lld"
+#else 
+typedef long JSON_int_t;
+#define JSON_PARSER_INTEGER_SSCANF_TOKEN "%ld"
+#define JSON_PARSER_INTEGER_SPRINTF_TOKEN "%ld"
+#endif
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif 
+
+typedef enum 
+{
+    JSON_T_NONE = 0,
+    JSON_T_ARRAY_BEGIN,  // 1
+    JSON_T_ARRAY_END,    // 2
+    JSON_T_OBJECT_BEGIN, // 3
+    JSON_T_OBJECT_END,   // 4
+    JSON_T_INTEGER,      // 5
+    JSON_T_FLOAT,        // 6
+    JSON_T_NULL,         // 7
+    JSON_T_TRUE,         // 8
+    JSON_T_FALSE,        // 9
+    JSON_T_STRING,       // 10
+    JSON_T_KEY,          // 11
+    JSON_T_MAX           // 12
+} JSON_type;
+
+typedef struct JSON_value_struct {
+    union {
+        JSON_int_t integer_value;
+        
+        long double float_value;
+        
+        struct {
+            const char* value;
+            size_t length;
+        } str;
+    } vu;
+} JSON_value;
+
+typedef struct JSON_parser_struct* JSON_parser;
+
+/*! \brief JSON parser callback 
+
+    \param ctx The pointer passed to new_JSON_parser.
+    \param type An element of JSON_type but not JSON_T_NONE.    
+    \param value A representation of the parsed value. This parameter is NULL for
+        JSON_T_ARRAY_BEGIN, JSON_T_ARRAY_END, JSON_T_OBJECT_BEGIN, JSON_T_OBJECT_END,
+        JSON_T_NULL, JSON_T_TRUE, and SON_T_FALSE. String values are always returned
+        as zero-terminated C strings.
+
+    \return Non-zero if parsing should continue, else zero.
+*/    
+typedef int (*JSON_parser_callback)(void* ctx, int type, const struct JSON_value_struct* value);
+
+
+/*! \brief The structure used to configure a JSON parser object 
+    
+    \param depth If negative, the parser can parse arbitrary levels of JSON, otherwise
+        the depth is the limit
+    \param Pointer to a callback. This parameter may be NULL. In this case the input is merely checked for validity.
+    \param Callback context. This parameter may be NULL.
+    \param depth. Specifies the levels of nested JSON to allow. Negative numbers yield unlimited nesting.
+    \param allowComments. To allow C style comments in JSON, set to non-zero.
+    \param handleFloatsManually. To decode floating point numbers manually set this parameter to non-zero.
+    
+    \return The parser object.
+*/
+typedef struct {
+    JSON_parser_callback     callback;
+    void*                    callback_ctx;
+    int                      depth;
+    int                      allow_comments;
+    int                      handle_floats_manually;
+} JSON_config;
+
+
+/*! \brief Initializes the JSON parser configuration structure to default values.
+
+    The default configuration is
+    - 127 levels of nested JSON (depends on JSON_PARSER_STACK_SIZE, see json_parser.c)
+    - no parsing, just checking for JSON syntax
+    - no comments
+
+    \param config. Used to configure the parser.
+*/
+JSON_PARSER_DLL_API void init_JSON_config(JSON_config* config);
+
+/*! \brief Create a JSON parser object 
+    
+    \param config. Used to configure the parser. Set to NULL to use the default configuration. 
+        See init_JSON_config
+    
+    \return The parser object.
+*/
+JSON_PARSER_DLL_API extern JSON_parser new_JSON_parser(JSON_config* config);
+
+/*! \brief Destroy a previously created JSON parser object. */
+JSON_PARSER_DLL_API extern void delete_JSON_parser(JSON_parser jc);
+
+/*! \brief Parse a character.
+
+    \return Non-zero, if all characters passed to this function are part of are valid JSON.
+*/
+JSON_PARSER_DLL_API extern int JSON_parser_char(JSON_parser jc, int next_char);
+
+/*! \brief Finalize parsing.
+
+    Call this method once after all input characters have been consumed.
+    
+    \return Non-zero, if all parsed characters are valid JSON, zero otherwise.
+*/
+JSON_PARSER_DLL_API extern int JSON_parser_done(JSON_parser jc);
+
+/*! \brief Determine if a given string is valid JSON white space 
+
+    \return Non-zero if the string is valid, zero otherwise.
+*/
+JSON_PARSER_DLL_API extern int JSON_parser_is_legal_white_space_string(const char* s);
+
+
+#ifdef __cplusplus
+}
+#endif 
+    
+
+#endif /* JSON_PARSER_H */
diff --git a/decoder/Makefile.am b/decoder/Makefile.am
new file mode 100644
index 00000000..a385197c
--- /dev/null
+++ b/decoder/Makefile.am
@@ -0,0 +1,84 @@
+bin_PROGRAMS = cdec
+
+if HAVE_GTEST
+noinst_PROGRAMS = \
+  dict_test \
+  weights_test \
+  trule_test \
+  hg_test \
+  ff_test \
+  logval_test \
+  parser_test \
+  grammar_test \
+  small_vector_test
+endif
+
+cdec_SOURCES = cdec.cc forest_writer.cc maxtrans_blunsom.cc cdec_ff.cc ff_factory.cc timing_stats.cc
+small_vector_test_SOURCES = small_vector_test.cc
+small_vector_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a
+parser_test_SOURCES = parser_test.cc
+parser_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a
+dict_test_SOURCES = dict_test.cc
+dict_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a
+ff_test_SOURCES = ff_test.cc
+ff_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a
+grammar_test_SOURCES = grammar_test.cc
+grammar_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a
+hg_test_SOURCES = hg_test.cc
+hg_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a
+trule_test_SOURCES = trule_test.cc
+trule_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a
+weights_test_SOURCES = weights_test.cc
+weights_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a
+logval_test_SOURCES = logval_test.cc
+logval_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS)
+
+LDADD = libcdec.a
+
+AM_CPPFLAGS = -W -Wall -Wno-sign-compare $(GTEST_CPPFLAGS)
+AM_LDFLAGS = -lz
+
+rule_lexer.cc: rule_lexer.l
+	$(LEX) -s -CF -8 -o$@ $<
+
+noinst_LIBRARIES = libcdec.a
+
+libcdec_a_SOURCES = \
+  rule_lexer.cc \
+  fst_translator.cc \
+  csplit.cc \
+  translator.cc \
+  scfg_translator.cc \
+  hg.cc \
+  hg_io.cc \
+  hg_intersect.cc \
+  viterbi.cc \
+  lattice.cc \
+  aligner.cc \
+  gzstream.cc \
+  apply_models.cc \
+  earley_composer.cc \
+  phrasetable_fst.cc \
+  sparse_vector.cc \
+  trule.cc \
+  filelib.cc \
+  stringlib.cc \
+  fdict.cc \
+  tdict.cc \
+  weights.cc \
+  ttables.cc \
+  ff.cc \
+  ff_lm.cc \
+  ff_wordalign.cc \
+  ff_csplit.cc \
+  ff_tagger.cc \
+  tromble_loss.cc \
+  freqdict.cc \
+  lexalign.cc \
+  lextrans.cc \
+  tagger.cc \
+  bottom_up_parser.cc \
+  phrasebased_translator.cc \
+  JSON_parser.c \
+  json_parse.cc \
+  grammar.cc
diff --git a/decoder/aligner.cc b/decoder/aligner.cc
new file mode 100644
index 00000000..bad97b74
--- /dev/null
+++ b/decoder/aligner.cc
@@ -0,0 +1,319 @@
+#include "aligner.h"
+
+#include "array2d.h"
+#include "hg.h"
+#include "sentence_metadata.h"
+#include "inside_outside.h"
+#include "viterbi.h"
+#include <set>
+
+using namespace std;
+
+static bool is_digit(char x) { return x >= '0' && x <= '9'; }
+
+boost::shared_ptr<Array2D<bool> > AlignerTools::ReadPharaohAlignmentGrid(const string& al) {
+  int max_x = 0;
+  int max_y = 0;
+  int i = 0;
+  size_t pos = al.rfind(" ||| ");
+  if (pos != string::npos) { i = pos + 5; }
+  while (i < al.size()) {
+    if (al[i] == '\n' || al[i] == '\r') break;
+    int x = 0;
+    while(i < al.size() && is_digit(al[i])) {
+      x *= 10;
+      x += al[i] - '0';
+      ++i;
+    }
+    if (x > max_x) max_x = x;
+    assert(i < al.size());
+    if(al[i] != '-') {
+      cerr << "BAD ALIGNMENT: " << al << endl;
+      abort();
+    }
+    ++i;
+    int y = 0;
+    while(i < al.size() && is_digit(al[i])) {
+      y *= 10;
+      y += al[i] - '0';
+      ++i;
+    }
+    if (y > max_y) max_y = y;
+    while(i < al.size() && al[i] == ' ') { ++i; }
+  }
+
+  boost::shared_ptr<Array2D<bool> > grid(new Array2D<bool>(max_x + 1, max_y + 1));
+  i = 0;
+  if (pos != string::npos) { i = pos + 5; }
+  while (i < al.size()) {
+    if (al[i] == '\n' || al[i] == '\r') break;
+    int x = 0;
+    while(i < al.size() && is_digit(al[i])) {
+      x *= 10;
+      x += al[i] - '0';
+      ++i;
+    }
+    assert(i < al.size());
+    assert(al[i] == '-');
+    ++i;
+    int y = 0;
+    while(i < al.size() && is_digit(al[i])) {
+      y *= 10;
+      y += al[i] - '0';
+      ++i;
+    }
+    (*grid)(x, y) = true;
+    while(i < al.size() && al[i] == ' ') { ++i; }
+  }
+  // cerr << *grid << endl;
+  return grid;
+}
+
+void AlignerTools::SerializePharaohFormat(const Array2D<bool>& alignment, ostream* out) {
+  bool need_space = false;
+  for (int i = 0; i < alignment.width(); ++i)
+    for (int j = 0; j < alignment.height(); ++j)
+      if (alignment(i,j)) {
+        if (need_space) (*out) << ' '; else need_space = true;
+        (*out) << i << '-' << j;
+      }
+  (*out) << endl;
+}
+
+// used with lexical models since they may not fully generate the
+// source string
+void SourceEdgeCoveragesUsingParseIndices(const Hypergraph& g,
+                                          vector<set<int> >* src_cov) {
+  src_cov->clear();
+  src_cov->resize(g.edges_.size());
+  
+  for (int i = 0; i < g.edges_.size(); ++i) {
+    const Hypergraph::Edge& edge = g.edges_[i];
+    set<int>& cov = (*src_cov)[i];
+    // no words
+    if (edge.rule_->EWords() == 0 || edge.rule_->FWords() == 0)
+      continue;
+    // aligned to NULL (crf ibm variant only)
+    if (edge.prev_i_ == -1 || edge.i_ == -1)
+      continue;
+    assert(edge.j_ >= 0);
+    assert(edge.prev_j_ >= 0);
+    if (edge.Arity() == 0) {
+      for (int k = edge.prev_i_; k < edge.prev_j_; ++k)
+        cov.insert(k);
+    } else {
+      // note: this code, which handles mixed NT and terminal
+      // rules assumes that nodes uniquely define a src and trg
+      // span.
+      int k = edge.prev_i_;
+      int j = 0;
+      const vector<WordID>& f = edge.rule_->e();  // rules are inverted
+      while (k < edge.prev_j_) {
+        if (f[j] > 0) {
+          cov.insert(k);
+          // cerr << "src: " << k << endl;
+          ++k;
+          ++j;
+        } else {
+          const Hypergraph::Node& tailnode = g.nodes_[edge.tail_nodes_[-f[j]]];
+          assert(tailnode.in_edges_.size() > 0);
+          // any edge will do:
+          const Hypergraph::Edge& rep_edge = g.edges_[tailnode.in_edges_.front()];
+          //cerr << "skip " << (rep_edge.prev_j_ - rep_edge.prev_i_) << endl;  // src span
+          k += (rep_edge.prev_j_ - rep_edge.prev_i_);  // src span
+          ++j;
+        }
+      }
+    }
+  }
+}
+
+int SourceEdgeCoveragesUsingTree(const Hypergraph& g,
+                                 int node_id,
+                                 int span_start,
+                                 vector<int>* spans,
+                                 vector<set<int> >* src_cov) {
+  const Hypergraph::Node& node = g.nodes_[node_id];
+  int k = -1;
+  for (int i = 0; i < node.in_edges_.size(); ++i) {
+    const int edge_id = node.in_edges_[i];
+    const Hypergraph::Edge& edge = g.edges_[edge_id];
+    set<int>& cov = (*src_cov)[edge_id];
+    const vector<WordID>& f = edge.rule_->e();  // rules are inverted
+    int j = 0;
+    k = span_start;
+    while (j < f.size()) {
+      if (f[j] > 0) {
+        cov.insert(k);
+        ++k;
+        ++j;
+      } else {
+        const int tail_node_id = edge.tail_nodes_[-f[j]];
+        int &right_edge = (*spans)[tail_node_id];
+        if (right_edge < 0)
+          right_edge = SourceEdgeCoveragesUsingTree(g, tail_node_id, k, spans, src_cov);
+        k = right_edge;
+        ++j;
+      }
+    }
+  }
+  return k;
+}
+
+void SourceEdgeCoveragesUsingTree(const Hypergraph& g,
+                                  vector<set<int> >* src_cov) {
+  src_cov->clear();
+  src_cov->resize(g.edges_.size());
+  vector<int> span_sizes(g.nodes_.size(), -1);
+  SourceEdgeCoveragesUsingTree(g, g.nodes_.size() - 1, 0, &span_sizes, src_cov);
+}
+
+int TargetEdgeCoveragesUsingTree(const Hypergraph& g,
+                                 int node_id,
+                                 int span_start,
+                                 vector<int>* spans,
+                                 vector<set<int> >* trg_cov) {
+  const Hypergraph::Node& node = g.nodes_[node_id];
+  int k = -1;
+  for (int i = 0; i < node.in_edges_.size(); ++i) {
+    const int edge_id = node.in_edges_[i];
+    const Hypergraph::Edge& edge = g.edges_[edge_id];
+    set<int>& cov = (*trg_cov)[edge_id];
+    int ntc = 0;
+    const vector<WordID>& e = edge.rule_->f();  // rules are inverted
+    int j = 0;
+    k = span_start;
+    while (j < e.size()) {
+      if (e[j] > 0) {
+        cov.insert(k);
+        ++k;
+        ++j;
+      } else {
+        const int tail_node_id = edge.tail_nodes_[ntc];
+        ++ntc;
+        int &right_edge = (*spans)[tail_node_id];
+        if (right_edge < 0)
+          right_edge = TargetEdgeCoveragesUsingTree(g, tail_node_id, k, spans, trg_cov);
+        k = right_edge;
+        ++j;
+      }
+    }
+    // cerr << "node=" << node_id << ": k=" << k << endl;
+  }
+  return k;
+}
+
+void TargetEdgeCoveragesUsingTree(const Hypergraph& g,
+                                  vector<set<int> >* trg_cov) {
+  trg_cov->clear();
+  trg_cov->resize(g.edges_.size());
+  vector<int> span_sizes(g.nodes_.size(), -1);
+  TargetEdgeCoveragesUsingTree(g, g.nodes_.size() - 1, 0, &span_sizes, trg_cov);
+}
+
+struct TransitionEventWeightFunction {
+  inline SparseVector<prob_t> operator()(const Hypergraph::Edge& e) const {
+    SparseVector<prob_t> result;
+    result.set_value(e.id_, e.edge_prob_);
+    return result;
+  }
+};
+
+// this code is rather complicated since it must deal with generating alignments
+// when lattices are specified as input as well as with models that do not generate
+// full sentence pairs (like lexical alignment models)
+void AlignerTools::WriteAlignment(const Lattice& src_lattice,
+                                  const Lattice& trg_lattice,
+                                  const Hypergraph& in_g,
+                                  ostream* out,
+                                  bool map_instead_of_viterbi,
+                                  const vector<bool>* edges) {
+  bool fix_up_src_spans = false;
+  const Hypergraph* g = &in_g;
+  if (!src_lattice.IsSentence() ||
+      !trg_lattice.IsSentence()) {
+    if (map_instead_of_viterbi) {
+      cerr << "  Lattice alignment: using Viterbi instead of MAP alignment\n";
+    }
+    map_instead_of_viterbi = false;
+    fix_up_src_spans = !src_lattice.IsSentence();
+  }
+  if (!map_instead_of_viterbi || edges) {
+    Hypergraph* new_hg = in_g.CreateViterbiHypergraph(edges);
+    for (int i = 0; i < new_hg->edges_.size(); ++i)
+      new_hg->edges_[i].edge_prob_ = prob_t::One();
+    g = new_hg;
+  }
+
+  vector<prob_t> edge_posteriors(g->edges_.size(), prob_t::Zero());
+  vector<WordID> trg_sent;
+  vector<WordID> src_sent;
+  if (fix_up_src_spans) {
+    ViterbiESentence(*g, &src_sent);
+  } else {
+    src_sent.resize(src_lattice.size());
+    for (int i = 0; i < src_sent.size(); ++i)
+      src_sent[i] = src_lattice[i][0].label;
+  }
+
+  ViterbiFSentence(*g, &trg_sent);
+
+  if (edges || !map_instead_of_viterbi) {
+    for (int i = 0; i < edge_posteriors.size(); ++i)
+      edge_posteriors[i] = prob_t::One();
+  } else { 
+    SparseVector<prob_t> posts;
+    const prob_t z = InsideOutside<prob_t, EdgeProb, SparseVector<prob_t>, TransitionEventWeightFunction>(*g, &posts);
+    for (int i = 0; i < edge_posteriors.size(); ++i)
+      edge_posteriors[i] = posts[i] / z;
+  }
+  vector<set<int> > src_cov(g->edges_.size());
+  vector<set<int> > trg_cov(g->edges_.size());
+  TargetEdgeCoveragesUsingTree(*g, &trg_cov);
+
+  if (fix_up_src_spans)
+    SourceEdgeCoveragesUsingTree(*g, &src_cov);
+  else
+    SourceEdgeCoveragesUsingParseIndices(*g, &src_cov);
+
+  // figure out the src and reference size;
+  int src_size = src_sent.size();
+  int ref_size = trg_sent.size();
+  Array2D<prob_t> align(src_size, ref_size, prob_t::Zero());
+  for (int c = 0; c < g->edges_.size(); ++c) {
+    const prob_t& p = edge_posteriors[c];
+    const set<int>& srcs = src_cov[c];
+    const set<int>& trgs = trg_cov[c];
+    for (set<int>::const_iterator si = srcs.begin();
+         si != srcs.end(); ++si) {
+      for (set<int>::const_iterator ti = trgs.begin();
+           ti != trgs.end(); ++ti) {
+        align(*si, *ti) += p;
+      }
+    }
+  }
+  if (g != &in_g) { delete g; g = NULL; }
+
+  prob_t threshold(0.9);
+  const bool use_soft_threshold = true; // TODO configure
+
+  Array2D<bool> grid(src_size, ref_size, false);
+  for (int j = 0; j < ref_size; ++j) {
+    if (use_soft_threshold) {
+      threshold = prob_t::Zero();
+      for (int i = 0; i < src_size; ++i)
+        if (align(i, j) > threshold) threshold = align(i, j);
+      //threshold *= prob_t(0.99);
+    }
+    for (int i = 0; i < src_size; ++i)
+      grid(i, j) = align(i, j) >= threshold;
+  }
+  if (out == &cout) {
+    // TODO need to do some sort of verbose flag
+    cerr << align << endl;
+    cerr << grid << endl;
+  }
+  (*out) << TD::GetString(src_sent) << " ||| " << TD::GetString(trg_sent) << " ||| ";
+  SerializePharaohFormat(grid, out);
+};
+
diff --git a/decoder/aligner.h b/decoder/aligner.h
new file mode 100644
index 00000000..cd159119
--- /dev/null
+++ b/decoder/aligner.h
@@ -0,0 +1,27 @@
+#ifndef _ALIGNER_H_
+
+#include <string>
+#include <iostream>
+#include <boost/shared_ptr.hpp>
+#include "array2d.h"
+#include "lattice.h"
+
+class Hypergraph;
+class SentenceMetadata;
+
+struct AlignerTools {
+  static boost::shared_ptr<Array2D<bool> > ReadPharaohAlignmentGrid(const std::string& al);
+  static void SerializePharaohFormat(const Array2D<bool>& alignment, std::ostream* out);
+
+  // assumption: g contains derivations of input/ref and
+  // ONLY input/ref.
+  // if edges is non-NULL, the alignment corresponding to the edge rules will be written
+  static void WriteAlignment(const Lattice& src,
+                             const Lattice& ref,
+                             const Hypergraph& g,
+                             std::ostream* out,
+                             bool map_instead_of_viterbi = true,
+                             const std::vector<bool>* edges = NULL);
+};
+
+#endif
diff --git a/decoder/apply_models.cc b/decoder/apply_models.cc
new file mode 100644
index 00000000..2908005f
--- /dev/null
+++ b/decoder/apply_models.cc
@@ -0,0 +1,426 @@
+#include "apply_models.h"
+
+#include <vector>
+#include <algorithm>
+#include <tr1/unordered_map>
+#include <tr1/unordered_set>
+
+#include <boost/functional/hash.hpp>
+
+#include "hg.h"
+#include "ff.h"
+
+using namespace std;
+using namespace std::tr1;
+
+struct Candidate;
+typedef SmallVector JVector;
+typedef vector<Candidate*> CandidateHeap;
+typedef vector<Candidate*> CandidateList;
+
+// default vector size (* sizeof string is memory used)
+static const size_t kRESERVE_NUM_NODES = 500000ul;
+
+// life cycle: candidates are created, placed on the heap
+// and retrieved by their estimated cost, when they're
+// retrieved, they're incorporated into the +LM hypergraph
+// where they also know the head node index they are
+// attached to.  After they are added to the +LM hypergraph
+// vit_prob_ and est_prob_ fields may be updated as better
+// derivations are found (this happens since the successor's
+// of derivation d may have a better score- they are
+// explored lazily).  However, the updates don't happen
+// when a candidate is in the heap so maintaining the heap
+// property is not an issue.
+struct Candidate {
+  int node_index_;                     // -1 until incorporated
+                                       // into the +LM forest
+  const Hypergraph::Edge* in_edge_;    // in -LM forest
+  Hypergraph::Edge out_edge_;
+  string state_;
+  const JVector j_;
+  prob_t vit_prob_;            // these are fixed until the cand
+                               // is popped, then they may be updated
+  prob_t est_prob_;
+
+  Candidate(const Hypergraph::Edge& e,
+            const JVector& j,
+            const Hypergraph& out_hg,
+            const vector<CandidateList>& D,
+            const vector<string>& node_states,
+            const SentenceMetadata& smeta,
+            const ModelSet& models,
+            bool is_goal) :
+      node_index_(-1),
+      in_edge_(&e),
+      j_(j) {
+    InitializeCandidate(out_hg, smeta, D, node_states, models, is_goal);
+  }
+
+  // used to query uniqueness
+  Candidate(const Hypergraph::Edge& e,
+            const JVector& j) : in_edge_(&e), j_(j) {}
+
+  bool IsIncorporatedIntoHypergraph() const {
+    return node_index_ >= 0;
+  }
+
+  void InitializeCandidate(const Hypergraph& out_hg,
+                           const SentenceMetadata& smeta,
+                           const vector<vector<Candidate*> >& D,
+                           const vector<string>& node_states,
+                           const ModelSet& models,
+                           const bool is_goal) {
+    const Hypergraph::Edge& in_edge = *in_edge_;
+    out_edge_.rule_ = in_edge.rule_;
+    out_edge_.feature_values_ = in_edge.feature_values_;
+    out_edge_.i_ = in_edge.i_;
+    out_edge_.j_ = in_edge.j_;
+    out_edge_.prev_i_ = in_edge.prev_i_;
+    out_edge_.prev_j_ = in_edge.prev_j_;
+    Hypergraph::TailNodeVector& tail = out_edge_.tail_nodes_;
+    tail.resize(j_.size());
+    prob_t p = prob_t::One();
+    // cerr << "\nEstimating application of " << in_edge.rule_->AsString() << endl;
+    for (int i = 0; i < tail.size(); ++i) {
+      const Candidate& ant = *D[in_edge.tail_nodes_[i]][j_[i]];
+      assert(ant.IsIncorporatedIntoHypergraph());
+      tail[i] = ant.node_index_;
+      p *= ant.vit_prob_;
+    }
+    prob_t edge_estimate = prob_t::One();
+    if (is_goal) {
+      assert(tail.size() == 1);
+      const string& ant_state = node_states[tail.front()];
+      models.AddFinalFeatures(ant_state, &out_edge_);
+    } else {
+      models.AddFeaturesToEdge(smeta, out_hg, node_states, &out_edge_, &state_, &edge_estimate);
+    }
+    vit_prob_ = out_edge_.edge_prob_ * p;
+    est_prob_ = vit_prob_ * edge_estimate;
+  }
+};
+
+ostream& operator<<(ostream& os, const Candidate& cand) {
+  os << "CAND[";
+  if (!cand.IsIncorporatedIntoHypergraph()) { os << "PENDING "; }
+  else { os << "+LM_node=" << cand.node_index_; }
+  os << " edge=" << cand.in_edge_->id_;
+  os << " j=<";
+  for (int i = 0; i < cand.j_.size(); ++i)
+    os << (i==0 ? "" : " ") << cand.j_[i];
+  os << "> vit=" << log(cand.vit_prob_);
+  os << " est=" << log(cand.est_prob_);
+  return os << ']';
+}
+
+struct HeapCandCompare {
+  bool operator()(const Candidate* l, const Candidate* r) const {
+    return l->est_prob_ < r->est_prob_;
+  }
+};
+
+struct EstProbSorter {
+  bool operator()(const Candidate* l, const Candidate* r) const {
+    return l->est_prob_ > r->est_prob_;
+  }
+};
+
+// the same candidate <edge, j> can be added multiple times if
+// j is multidimensional (if you're going NW in Manhattan, you
+// can first go north, then west, or you can go west then north)
+// this is a hash function on the relevant variables from
+// Candidate to enforce this.
+struct CandidateUniquenessHash {
+  size_t operator()(const Candidate* c) const {
+    size_t x = 5381;
+    x = ((x << 5) + x) ^ c->in_edge_->id_;
+    for (int i = 0; i < c->j_.size(); ++i)
+      x = ((x << 5) + x) ^ c->j_[i];
+    return x;
+  }
+};
+
+struct CandidateUniquenessEquals {
+  bool operator()(const Candidate* a, const Candidate* b) const {
+    return (a->in_edge_ == b->in_edge_) && (a->j_ == b->j_);
+  }
+};
+
+typedef unordered_set<const Candidate*, CandidateUniquenessHash, CandidateUniquenessEquals> UniqueCandidateSet;
+typedef unordered_map<string, Candidate*, boost::hash<string> > State2Node;
+
+class CubePruningRescorer {
+
+public:
+  CubePruningRescorer(const ModelSet& m,
+                      const SentenceMetadata& sm,
+                      const Hypergraph& i,
+                      int pop_limit,
+                      Hypergraph* o) :
+      models(m),
+      smeta(sm),
+      in(i),
+      out(*o),
+      D(in.nodes_.size()),
+      pop_limit_(pop_limit) {
+    cerr << "  Applying feature functions (cube pruning, pop_limit = " << pop_limit_ << ')' << endl;
+    node_states_.reserve(kRESERVE_NUM_NODES);
+  }
+
+  void Apply() {
+    int num_nodes = in.nodes_.size();
+    int goal_id = num_nodes - 1;
+    int pregoal = goal_id - 1;
+    int every = 1;
+    if (num_nodes > 100) every = 10;
+    assert(in.nodes_[pregoal].out_edges_.size() == 1);
+    cerr << "    ";
+    for (int i = 0; i < in.nodes_.size(); ++i) {
+      if (i % every == 0) cerr << '.';
+      KBest(i, i == goal_id);
+    }
+    cerr << endl;
+    cerr << "  Best path: " << log(D[goal_id].front()->vit_prob_)
+         << "\t" << log(D[goal_id].front()->est_prob_) << endl;
+    out.PruneUnreachable(D[goal_id].front()->node_index_);
+    FreeAll();
+  }
+
+ private:
+  void FreeAll() {
+    for (int i = 0; i < D.size(); ++i) {
+      CandidateList& D_i = D[i];
+      for (int j = 0; j < D_i.size(); ++j)
+        delete D_i[j];
+    }
+    D.clear();
+  }
+
+  void IncorporateIntoPlusLMForest(Candidate* item, State2Node* s2n, CandidateList* freelist) {
+    Hypergraph::Edge* new_edge = out.AddEdge(item->out_edge_.rule_, item->out_edge_.tail_nodes_);
+    new_edge->feature_values_ = item->out_edge_.feature_values_;
+    new_edge->edge_prob_ = item->out_edge_.edge_prob_;
+    new_edge->i_ = item->out_edge_.i_;
+    new_edge->j_ = item->out_edge_.j_;
+    new_edge->prev_i_ = item->out_edge_.prev_i_;
+    new_edge->prev_j_ = item->out_edge_.prev_j_;
+    Candidate*& o_item = (*s2n)[item->state_];
+    if (!o_item) o_item = item;
+    
+    int& node_id = o_item->node_index_;
+    if (node_id < 0) {
+      Hypergraph::Node* new_node = out.AddNode(in.nodes_[item->in_edge_->head_node_].cat_);
+      node_states_.push_back(item->state_);
+      node_id = new_node->id_;
+    }
+    Hypergraph::Node* node = &out.nodes_[node_id];
+    out.ConnectEdgeToHeadNode(new_edge, node);
+
+    // update candidate if we have a better derivation
+    // note: the difference between the vit score and the estimated
+    // score is the same for all items with a common residual DP
+    // state
+    if (item->vit_prob_ > o_item->vit_prob_) {
+      assert(o_item->state_ == item->state_);    // sanity check!
+      o_item->est_prob_ = item->est_prob_;
+      o_item->vit_prob_ = item->vit_prob_;
+    }
+    if (item != o_item) freelist->push_back(item);
+  }
+
+  void KBest(const int vert_index, const bool is_goal) {
+    // cerr << "KBest(" << vert_index << ")\n";
+    CandidateList& D_v = D[vert_index];
+    assert(D_v.empty());
+    const Hypergraph::Node& v = in.nodes_[vert_index];
+    // cerr << "  has " << v.in_edges_.size() << " in-coming edges\n";
+    const vector<int>& in_edges = v.in_edges_;
+    CandidateHeap cand;
+    CandidateList freelist;
+    cand.reserve(in_edges.size());
+    UniqueCandidateSet unique_cands;
+    for (int i = 0; i < in_edges.size(); ++i) {
+      const Hypergraph::Edge& edge = in.edges_[in_edges[i]];
+      const JVector j(edge.tail_nodes_.size(), 0);
+      cand.push_back(new Candidate(edge, j, out, D, node_states_, smeta, models, is_goal));
+      assert(unique_cands.insert(cand.back()).second);  // these should all be unique!
+    }
+//    cerr << "  making heap of " << cand.size() << " candidates\n";
+    make_heap(cand.begin(), cand.end(), HeapCandCompare());
+    State2Node state2node;   // "buf" in Figure 2
+    int pops = 0;
+    while(!cand.empty() && pops < pop_limit_) {
+      pop_heap(cand.begin(), cand.end(), HeapCandCompare());
+      Candidate* item = cand.back();
+      cand.pop_back();
+      // cerr << "POPPED: " << *item << endl;
+      PushSucc(*item, is_goal, &cand, &unique_cands);
+      IncorporateIntoPlusLMForest(item, &state2node, &freelist);
+      ++pops;
+    }
+    D_v.resize(state2node.size());
+    int c = 0;
+    for (State2Node::iterator i = state2node.begin(); i != state2node.end(); ++i)
+      D_v[c++] = i->second;
+    sort(D_v.begin(), D_v.end(), EstProbSorter());
+    // cerr << "  expanded to " << D_v.size() << " nodes\n";
+
+    for (int i = 0; i < cand.size(); ++i)
+      delete cand[i];
+    // freelist is necessary since even after an item merged, it still stays in
+    // the unique set so it can't be deleted til now
+    for (int i = 0; i < freelist.size(); ++i)
+      delete freelist[i];
+  }
+
+  void PushSucc(const Candidate& item, const bool is_goal, CandidateHeap* pcand, UniqueCandidateSet* cs) {
+    CandidateHeap& cand = *pcand;
+    for (int i = 0; i < item.j_.size(); ++i) {
+      JVector j = item.j_;
+      ++j[i];
+      if (j[i] < D[item.in_edge_->tail_nodes_[i]].size()) {
+        Candidate query_unique(*item.in_edge_, j);
+        if (cs->count(&query_unique) == 0) {
+          Candidate* new_cand = new Candidate(*item.in_edge_, j, out, D, node_states_, smeta, models, is_goal);
+          cand.push_back(new_cand);
+          push_heap(cand.begin(), cand.end(), HeapCandCompare());
+          assert(cs->insert(new_cand).second);  // insert into uniqueness set, sanity check
+        }
+      }
+    }
+  }
+
+  const ModelSet& models;
+  const SentenceMetadata& smeta;
+  const Hypergraph& in;
+  Hypergraph& out;
+
+  vector<CandidateList> D;   // maps nodes in in-HG to the
+                             // equivalent nodes (many due to state
+                             // splits) in the out-HG.
+  vector<string> node_states_;  // for each node in the out-HG what is
+                             // its q function value?
+  const int pop_limit_;
+};
+
+struct NoPruningRescorer {
+  NoPruningRescorer(const ModelSet& m, const SentenceMetadata &sm, const Hypergraph& i, Hypergraph* o) :
+      models(m),
+      smeta(sm),
+      in(i),
+      out(*o),
+      nodemap(i.nodes_.size()) {
+    cerr << "  Rescoring forest (full intersection)\n";
+    node_states_.reserve(kRESERVE_NUM_NODES);
+  }
+
+  typedef unordered_map<string, int, boost::hash<string> > State2NodeIndex;
+
+  void ExpandEdge(const Hypergraph::Edge& in_edge, bool is_goal, State2NodeIndex* state2node) {
+    const int arity = in_edge.Arity();
+    Hypergraph::TailNodeVector ends(arity);
+    for (int i = 0; i < arity; ++i)
+      ends[i] = nodemap[in_edge.tail_nodes_[i]].size();
+
+    Hypergraph::TailNodeVector tail_iter(arity, 0);
+    bool done = false;
+    while (!done) {
+      Hypergraph::TailNodeVector tail(arity);
+      for (int i = 0; i < arity; ++i)
+        tail[i] = nodemap[in_edge.tail_nodes_[i]][tail_iter[i]];
+      Hypergraph::Edge* new_edge = out.AddEdge(in_edge.rule_, tail);
+      new_edge->feature_values_ = in_edge.feature_values_;
+      new_edge->i_ = in_edge.i_;
+      new_edge->j_ = in_edge.j_;
+      new_edge->prev_i_ = in_edge.prev_i_;
+      new_edge->prev_j_ = in_edge.prev_j_;
+      string head_state;
+      if (is_goal) {
+        assert(tail.size() == 1);
+        const string& ant_state = node_states_[tail.front()];
+        models.AddFinalFeatures(ant_state, new_edge);
+      } else {
+        prob_t edge_estimate; // this is a full intersection, so we disregard this
+        models.AddFeaturesToEdge(smeta, out, node_states_, new_edge, &head_state, &edge_estimate);
+      }
+      int& head_plus1 = (*state2node)[head_state];
+      if (!head_plus1) {
+        head_plus1 = out.AddNode(in_edge.rule_->GetLHS())->id_ + 1;
+        node_states_.push_back(head_state);
+        nodemap[in_edge.head_node_].push_back(head_plus1 - 1);
+      }
+      const int head_index = head_plus1 - 1;
+      out.ConnectEdgeToHeadNode(new_edge->id_, head_index);
+
+      int ii = 0;
+      for (; ii < arity; ++ii) {
+        ++tail_iter[ii];
+        if (tail_iter[ii] < ends[ii]) break;
+        tail_iter[ii] = 0;
+      }
+      done = (ii == arity);
+    }
+  }
+
+  void ProcessOneNode(const int node_num, const bool is_goal) {
+    State2NodeIndex state2node;
+    const Hypergraph::Node& node = in.nodes_[node_num];
+    for (int i = 0; i < node.in_edges_.size(); ++i) {
+      const Hypergraph::Edge& edge = in.edges_[node.in_edges_[i]];
+      ExpandEdge(edge, is_goal, &state2node);
+    }
+  }
+
+  void Apply() {
+    int num_nodes = in.nodes_.size();
+    int goal_id = num_nodes - 1;
+    int pregoal = goal_id - 1;
+    int every = 1;
+    if (num_nodes > 100) every = 10;
+    assert(in.nodes_[pregoal].out_edges_.size() == 1);
+    cerr << "    ";
+    for (int i = 0; i < in.nodes_.size(); ++i) {
+      if (i % every == 0) cerr << '.';
+      ProcessOneNode(i, i == goal_id);
+    }
+    cerr << endl;
+  }
+
+ private:
+  const ModelSet& models;
+  const SentenceMetadata& smeta;
+  const Hypergraph& in;
+  Hypergraph& out;
+
+  vector<vector<int> > nodemap;
+  vector<string> node_states_;  // for each node in the out-HG what is
+                             // its q function value?
+};
+
+// each node in the graph has one of these, it keeps track of
+void ApplyModelSet(const Hypergraph& in,
+                   const SentenceMetadata& smeta,
+                   const ModelSet& models,
+                   const IntersectionConfiguration& config,
+                   Hypergraph* out) {
+  // TODO special handling when all models are stateless
+  if (config.algorithm == 1) {
+    int pl = config.pop_limit;
+    if (pl > 100 && in.nodes_.size() > 80000) {
+      cerr << "  Note: reducing pop_limit to " << pl << " for very large forest\n";
+      pl = 30;
+    }
+    CubePruningRescorer ma(models, smeta, in, pl, out);
+    ma.Apply();
+  } else if (config.algorithm == 0) {
+    NoPruningRescorer ma(models, smeta, in, out);
+    ma.Apply();
+  } else {
+    cerr << "Don't understand intersection algorithm " << config.algorithm << endl;
+    exit(1);
+  }
+  out->is_linear_chain_ = in.is_linear_chain_;  // TODO remove when this is computed
+                                                // automatically
+}
+
diff --git a/decoder/apply_models.h b/decoder/apply_models.h
new file mode 100644
index 00000000..d6d8b34a
--- /dev/null
+++ b/decoder/apply_models.h
@@ -0,0 +1,20 @@
+#ifndef _APPLY_MODELS_H_
+#define _APPLY_MODELS_H_
+
+struct ModelSet;
+struct Hypergraph;
+struct SentenceMetadata;
+
+struct IntersectionConfiguration {
+  const int algorithm; // 0 = full intersection, 1 = cube pruning
+  const int pop_limit; // max number of pops off the heap at each node
+  IntersectionConfiguration(int alg, int k) : algorithm(alg), pop_limit(k) {}
+};
+
+void ApplyModelSet(const Hypergraph& in,
+                   const SentenceMetadata& smeta,
+                   const ModelSet& models,
+                   const IntersectionConfiguration& config,
+                   Hypergraph* out);
+
+#endif
diff --git a/decoder/array2d.h b/decoder/array2d.h
new file mode 100644
index 00000000..e63eda0d
--- /dev/null
+++ b/decoder/array2d.h
@@ -0,0 +1,172 @@
+#ifndef ARRAY2D_H_
+#define ARRAY2D_H_
+
+#include <iostream>
+#include <algorithm>
+#include <cassert>
+#include <vector>
+#include <string>
+
+template<typename T>
+class Array2D {
+ public:
+  typedef typename std::vector<T>::reference reference;
+  typedef typename std::vector<T>::const_reference const_reference;
+  typedef typename std::vector<T>::iterator iterator;
+  typedef typename std::vector<T>::const_iterator const_iterator;
+  Array2D() : width_(0), height_(0) {}
+  Array2D(int w, int h, const T& d = T()) :
+    width_(w), height_(h), data_(w*h, d) {}
+  Array2D(const Array2D& rhs) :
+    width_(rhs.width_), height_(rhs.height_), data_(rhs.data_) {}
+  bool empty() const { return data_.empty(); }
+  void resize(int w, int h, const T& d = T()) {
+    data_.resize(w * h, d);
+    width_ = w;
+    height_ = h;
+  }
+  const Array2D& operator=(const Array2D& rhs) {
+    data_ = rhs.data_;
+    width_ = rhs.width_;
+    height_ = rhs.height_;
+    return *this;
+  }
+  void fill(const T& v) { data_.assign(data_.size(), v); }
+  int width() const { return width_; }
+  int height() const { return height_; }
+  reference operator()(int i, int j) {
+    return data_[offset(i, j)];
+  }
+  void clear() { data_.clear(); width_=0; height_=0; }
+  const_reference operator()(int i, int j) const {
+    return data_[offset(i, j)];
+  }
+  iterator begin_col(int j) {
+    return data_.begin() + offset(0,j);
+  }
+  const_iterator begin_col(int j) const {
+    return data_.begin() + offset(0,j);
+  }
+  iterator end_col(int j) {
+    return data_.begin() + offset(0,j) + width_;
+  }
+  const_iterator end_col(int j) const {
+    return data_.begin() + offset(0,j) + width_;
+  }
+  iterator end() { return data_.end(); }
+  const_iterator end() const { return data_.end(); }
+  const Array2D<T>& operator*=(const T& x) {
+    std::transform(data_.begin(), data_.end(), data_.begin(),
+        std::bind2nd(std::multiplies<T>(), x));
+  }
+  const Array2D<T>& operator/=(const T& x) {
+    std::transform(data_.begin(), data_.end(), data_.begin(),
+        std::bind2nd(std::divides<T>(), x));
+  }
+  const Array2D<T>& operator+=(const Array2D<T>& m) {
+    std::transform(m.data_.begin(), m.data_.end(), data_.begin(), data_.begin(), std::plus<T>());
+  }
+  const Array2D<T>& operator-=(const Array2D<T>& m) {
+    std::transform(m.data_.begin(), m.data_.end(), data_.begin(), data_.begin(), std::minus<T>());
+  }
+
+ private:
+  inline int offset(int i, int j) const {
+    assert(i<width_);
+    assert(j<height_);
+    return i + j * width_;
+  }
+
+  int width_;
+  int height_;
+
+  std::vector<T> data_;
+};
+
+template <typename T>
+Array2D<T> operator*(const Array2D<T>& l, const T& scalar) {
+  Array2D<T> res(l);
+  res *= scalar;
+  return res;
+}
+
+template <typename T>
+Array2D<T> operator*(const T& scalar, const Array2D<T>& l) {
+  Array2D<T> res(l);
+  res *= scalar;
+  return res;
+}
+
+template <typename T>
+Array2D<T> operator/(const Array2D<T>& l, const T& scalar) {
+  Array2D<T> res(l);
+  res /= scalar;
+  return res;
+}
+
+template <typename T>
+Array2D<T> operator+(const Array2D<T>& l, const Array2D<T>& r) {
+  Array2D<T> res(l);
+  res += r;
+  return res;
+}
+
+template <typename T>
+Array2D<T> operator-(const Array2D<T>& l, const Array2D<T>& r) {
+  Array2D<T> res(l);
+  res -= r;
+  return res;
+}
+
+template <typename T>
+inline std::ostream& operator<<(std::ostream& os, const Array2D<T>& m) {
+  for (int i=0; i<m.width(); ++i) {
+    for (int j=0; j<m.height(); ++j)
+      os << '\t' << m(i,j);
+    os << '\n';
+  }
+  return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const Array2D<bool>& m) {
+  os << ' ';
+  for (int j=0; j<m.height(); ++j)
+    os << (j%10);
+  os << "\n";
+  for (int i=0; i<m.width(); ++i) {
+    os << (i%10);
+    for (int j=0; j<m.height(); ++j)
+      os << (m(i,j) ? '*' : '.');
+    os << (i%10) << "\n";
+  }
+  os << ' ';
+  for (int j=0; j<m.height(); ++j)
+    os << (j%10);
+  os << "\n";
+  return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const Array2D<std::vector<bool> >& m) {
+  os << ' ';
+  for (int j=0; j<m.height(); ++j)
+    os << (j%10) << "\t";
+  os << "\n";
+  for (int i=0; i<m.width(); ++i) {
+    os << (i%10);
+    for (int j=0; j<m.height(); ++j) {
+      const std::vector<bool>& ar = m(i,j);
+      for (int k=0; k<ar.size(); ++k)
+        os << (ar[k] ? '*' : '.');
+    }
+    os << "\t";
+    os << (i%10) << "\n";
+  }
+  os << ' ';
+  for (int j=0; j<m.height(); ++j)
+    os << (j%10) << "\t";
+  os << "\n";
+  return os;
+}
+
+#endif
+
diff --git a/decoder/bottom_up_parser.cc b/decoder/bottom_up_parser.cc
new file mode 100644
index 00000000..dd54a606
--- /dev/null
+++ b/decoder/bottom_up_parser.cc
@@ -0,0 +1,302 @@
+#include "bottom_up_parser.h"
+
+#include <iostream>
+#include <map>
+
+#include "hg.h"
+#include "array2d.h"
+#include "tdict.h"
+
+using namespace std;
+
+struct ParserStats {
+  ParserStats() : active_items(), passive_items() {}
+  void Reset() { active_items=0; passive_items=0; }
+  void Report() {
+    cerr << "  ACTIVE ITEMS: " << active_items << "\tPASSIVE ITEMS: " << passive_items << endl;
+  }
+  int active_items;
+  int passive_items;
+  void NotifyActive(int i, int j) { ++active_items; }
+  void NotifyPassive(int i, int j) { ++passive_items; }
+};
+
+ParserStats stats;
+
+class ActiveChart;
+class PassiveChart {
+ public:
+  PassiveChart(const string& goal,
+               const vector<GrammarPtr>& grammars,
+               const Lattice& input,
+               Hypergraph* forest);
+  ~PassiveChart();
+
+  inline const vector<int>& operator()(int i, int j) const { return chart_(i,j); }
+  bool Parse();
+  inline int size() const { return chart_.width(); }
+  inline bool GoalFound() const { return goal_idx_ >= 0; }
+  inline int GetGoalIndex() const { return goal_idx_; }
+
+ private:
+  void ApplyRules(const int i,
+                  const int j,
+                  const RuleBin* rules,
+                  const Hypergraph::TailNodeVector& tail,
+                  const float lattice_cost);
+
+  void ApplyRule(const int i,
+                 const int j,
+                 const TRulePtr& r,
+                 const Hypergraph::TailNodeVector& ant_nodes,
+                 const float lattice_cost);
+
+  void ApplyUnaryRules(const int i, const int j);
+
+  const vector<GrammarPtr>& grammars_;
+  const Lattice& input_;
+  Hypergraph* forest_;
+  Array2D<vector<int> > chart_;   // chart_(i,j) is the list of nodes derived spanning i,j
+  typedef map<int, int> Cat2NodeMap;
+  Array2D<Cat2NodeMap> nodemap_;
+  vector<ActiveChart*> act_chart_;
+  const WordID goal_cat_;    // category that is being searched for at [0,n]
+  TRulePtr goal_rule_;
+  int goal_idx_;             // index of goal node, if found
+  const int lc_fid_;
+
+  static WordID kGOAL;       // [Goal]
+};
+
+WordID PassiveChart::kGOAL = 0;
+
+class ActiveChart {
+ public:
+  ActiveChart(const Hypergraph* hg, const PassiveChart& psv_chart) :
+    hg_(hg),
+    act_chart_(psv_chart.size(), psv_chart.size()), psv_chart_(psv_chart) {}
+
+  struct ActiveItem {
+    ActiveItem(const GrammarIter* g, const Hypergraph::TailNodeVector& a, float lcost) :
+      gptr_(g), ant_nodes_(a), lattice_cost(lcost) {}
+    explicit ActiveItem(const GrammarIter* g) :
+      gptr_(g), ant_nodes_(), lattice_cost(0.0) {}
+
+    void ExtendTerminal(int symbol, float src_cost, vector<ActiveItem>* out_cell) const {
+      const GrammarIter* ni = gptr_->Extend(symbol);
+      if (ni) {
+        stats.NotifyActive(-1,-1);  // TRACKING STATS
+        out_cell->push_back(ActiveItem(ni, ant_nodes_, lattice_cost + src_cost));
+      }
+    }
+    void ExtendNonTerminal(const Hypergraph* hg, int node_index, vector<ActiveItem>* out_cell) const {
+      int symbol = hg->nodes_[node_index].cat_;
+      const GrammarIter* ni = gptr_->Extend(symbol);
+      if (!ni) return;
+      stats.NotifyActive(-1,-1);  // TRACKING STATS
+      Hypergraph::TailNodeVector na(ant_nodes_.size() + 1);
+      for (int i = 0; i < ant_nodes_.size(); ++i)
+        na[i] = ant_nodes_[i];
+      na[ant_nodes_.size()] = node_index;
+      out_cell->push_back(ActiveItem(ni, na, lattice_cost));
+    }
+
+    const GrammarIter* gptr_;
+    Hypergraph::TailNodeVector ant_nodes_;
+    float lattice_cost;  // TODO? use SparseVector<double>
+  };
+
+  inline const vector<ActiveItem>& operator()(int i, int j) const { return act_chart_(i,j); }
+  void SeedActiveChart(const Grammar& g) {
+    int size = act_chart_.width();
+    for (int i = 0; i < size; ++i)
+      if (g.HasRuleForSpan(i,i,0))
+        act_chart_(i,i).push_back(ActiveItem(g.GetRoot()));
+  }
+
+  void ExtendActiveItems(int i, int k, int j) {
+    //cerr << "  LOOK(" << i << "," << k << ") for completed items in (" << k << "," << j << ")\n";
+    vector<ActiveItem>& cell = act_chart_(i,j);
+    const vector<ActiveItem>& icell = act_chart_(i,k);
+    const vector<int>& idxs = psv_chart_(k, j);
+    //if (!idxs.empty()) { cerr << "FOUND IN (" << k << "," << j << ")\n"; }
+    for (vector<ActiveItem>::const_iterator di = icell.begin(); di != icell.end(); ++di) {
+      for (vector<int>::const_iterator ni = idxs.begin(); ni != idxs.end(); ++ni) {
+         di->ExtendNonTerminal(hg_, *ni, &cell);
+      }
+    }
+  }
+
+  void AdvanceDotsForAllItemsInCell(int i, int j, const vector<vector<LatticeArc> >& input) {
+    //cerr << "ADVANCE(" << i << "," << j << ")\n";
+    for (int k=i+1; k < j; ++k)
+      ExtendActiveItems(i, k, j);
+
+    const vector<LatticeArc>& out_arcs = input[j-1];
+    for (vector<LatticeArc>::const_iterator ai = out_arcs.begin();
+         ai != out_arcs.end(); ++ai) {
+      const WordID& f = ai->label;
+      const double& c = ai->cost;
+      const int& len = ai->dist2next;
+      //VLOG(1) << "F: " << TD::Convert(f) << endl;
+      const vector<ActiveItem>& ec = act_chart_(i, j-1);
+      for (vector<ActiveItem>::const_iterator di = ec.begin(); di != ec.end(); ++di)
+        di->ExtendTerminal(f, c, &act_chart_(i, j + len - 1));
+    }
+  }
+
+ private:
+  const Hypergraph* hg_;
+  Array2D<vector<ActiveItem> > act_chart_;
+  const PassiveChart& psv_chart_; 
+};
+
+PassiveChart::PassiveChart(const string& goal,
+                           const vector<GrammarPtr>& grammars,
+                           const Lattice& input,
+                           Hypergraph* forest) :
+    grammars_(grammars),
+    input_(input),
+    forest_(forest),
+    chart_(input.size()+1, input.size()+1),
+    nodemap_(input.size()+1, input.size()+1),
+    goal_cat_(TD::Convert(goal) * -1),
+    goal_rule_(new TRule("[Goal] ||| [" + goal + ",1] ||| [" + goal + ",1]")),
+    goal_idx_(-1),
+    lc_fid_(FD::Convert("LatticeCost")) {
+  act_chart_.resize(grammars_.size());
+  for (int i = 0; i < grammars_.size(); ++i)
+    act_chart_[i] = new ActiveChart(forest, *this);
+  if (!kGOAL) kGOAL = TD::Convert("Goal") * -1;
+  cerr << "  Goal category: [" << goal << ']' << endl;
+}
+
+void PassiveChart::ApplyRule(const int i,
+                             const int j,
+                             const TRulePtr& r,
+                             const Hypergraph::TailNodeVector& ant_nodes,
+                             const float lattice_cost) {
+  stats.NotifyPassive(i,j);  // TRACKING STATS
+  Hypergraph::Edge* new_edge = forest_->AddEdge(r, ant_nodes);
+  new_edge->prev_i_ = r->prev_i;
+  new_edge->prev_j_ = r->prev_j;
+  new_edge->i_ = i;
+  new_edge->j_ = j;
+  new_edge->feature_values_ = r->GetFeatureValues();
+  if (lattice_cost)
+    new_edge->feature_values_.set_value(lc_fid_, lattice_cost);
+  Cat2NodeMap& c2n = nodemap_(i,j);
+  const bool is_goal = (r->GetLHS() == kGOAL);
+  const Cat2NodeMap::iterator ni = c2n.find(r->GetLHS());
+  Hypergraph::Node* node = NULL;
+  if (ni == c2n.end()) {
+    node = forest_->AddNode(r->GetLHS());
+    c2n[r->GetLHS()] = node->id_;
+    if (is_goal) {
+      assert(goal_idx_ == -1);
+      goal_idx_ = node->id_;
+    } else {
+      chart_(i,j).push_back(node->id_);
+    }
+  } else {
+    node = &forest_->nodes_[ni->second];
+  }
+  forest_->ConnectEdgeToHeadNode(new_edge, node);
+}
+
+void PassiveChart::ApplyRules(const int i,
+                       const int j,
+                       const RuleBin* rules,
+                       const Hypergraph::TailNodeVector& tail,
+                       const float lattice_cost) {
+  const int n = rules->GetNumRules();
+  for (int k = 0; k < n; ++k)
+    ApplyRule(i, j, rules->GetIthRule(k), tail, lattice_cost);
+}
+
+void PassiveChart::ApplyUnaryRules(const int i, const int j) {
+  const vector<int>& nodes = chart_(i,j);  // reference is important!
+  for (int gi = 0; gi < grammars_.size(); ++gi) {
+    if (!grammars_[gi]->HasRuleForSpan(i,j,input_.Distance(i,j))) continue;
+    for (int di = 0; di < nodes.size(); ++di) {
+      const WordID& cat = forest_->nodes_[nodes[di]].cat_;
+      const vector<TRulePtr>& unaries = grammars_[gi]->GetUnaryRulesForRHS(cat);
+      for (int ri = 0; ri < unaries.size(); ++ri) {
+        // cerr << "At (" << i << "," << j << "): applying " << unaries[ri]->AsString() << endl;
+        const Hypergraph::TailNodeVector ant(1, nodes[di]);
+        ApplyRule(i, j, unaries[ri], ant, 0);  // may update nodes
+      }
+    }
+  }
+}
+
+bool PassiveChart::Parse() {
+  forest_->nodes_.reserve(input_.size() * input_.size() * 2);
+  forest_->edges_.reserve(input_.size() * input_.size() * 1000);  // TODO: reservation??
+  goal_idx_ = -1;
+  for (int gi = 0; gi < grammars_.size(); ++gi)
+    act_chart_[gi]->SeedActiveChart(*grammars_[gi]);
+
+  cerr << "    ";
+  for (int l=1; l<input_.size()+1; ++l) {
+    cerr << '.';
+    for (int i=0; i<input_.size() + 1 - l; ++i) {
+      int j = i + l;
+      for (int gi = 0; gi < grammars_.size(); ++gi) {
+        const Grammar& g = *grammars_[gi];
+        if (g.HasRuleForSpan(i, j, input_.Distance(i, j))) {
+          act_chart_[gi]->AdvanceDotsForAllItemsInCell(i, j, input_);
+
+          const vector<ActiveChart::ActiveItem>& cell = (*act_chart_[gi])(i,j);
+          for (vector<ActiveChart::ActiveItem>::const_iterator ai = cell.begin();
+               ai != cell.end(); ++ai) {
+            const RuleBin* rules = (ai->gptr_->GetRules());
+            if (!rules) continue;
+            ApplyRules(i, j, rules, ai->ant_nodes_, ai->lattice_cost);
+          }
+        }
+      }
+      ApplyUnaryRules(i,j);
+
+      for (int gi = 0; gi < grammars_.size(); ++gi) {
+        const Grammar& g = *grammars_[gi];
+          // deal with non-terminals that were just proved
+          if (g.HasRuleForSpan(i, j, input_.Distance(i,j)))
+            act_chart_[gi]->ExtendActiveItems(i, i, j);
+      }
+    }
+    const vector<int>& dh = chart_(0, input_.size());
+    for (int di = 0; di < dh.size(); ++di) {
+      const Hypergraph::Node& node = forest_->nodes_[dh[di]];
+      if (node.cat_ == goal_cat_) {
+        Hypergraph::TailNodeVector ant(1, node.id_);
+        ApplyRule(0, input_.size(), goal_rule_, ant, 0);
+      }
+    }
+  }
+  cerr << endl;
+
+  if (GoalFound())
+    forest_->PruneUnreachable(forest_->nodes_.size() - 1);
+  return GoalFound();
+}
+
+PassiveChart::~PassiveChart() {
+  for (int i = 0; i < act_chart_.size(); ++i)
+    delete act_chart_[i];
+}
+
+ExhaustiveBottomUpParser::ExhaustiveBottomUpParser(
+    const string& goal_sym,
+    const vector<GrammarPtr>& grammars) :
+  goal_sym_(goal_sym),
+  grammars_(grammars) {}
+
+bool ExhaustiveBottomUpParser::Parse(const Lattice& input,
+                                     Hypergraph* forest) const {
+  stats.Reset();
+  PassiveChart chart(goal_sym_, grammars_, input, forest);
+  const bool result = chart.Parse();
+  stats.Report();
+  return result;
+}
diff --git a/decoder/bottom_up_parser.h b/decoder/bottom_up_parser.h
new file mode 100644
index 00000000..546bfb54
--- /dev/null
+++ b/decoder/bottom_up_parser.h
@@ -0,0 +1,27 @@
+#ifndef _BOTTOM_UP_PARSER_H_
+#define _BOTTOM_UP_PARSER_H_
+
+#include <vector>
+#include <string>
+
+#include "lattice.h"
+#include "grammar.h"
+
+class Hypergraph;
+
+class ExhaustiveBottomUpParser {
+ public:
+  ExhaustiveBottomUpParser(const std::string& goal_sym,
+                           const std::vector<GrammarPtr>& grammars);
+
+  // returns true if goal reached spanning the full input
+  // forest contains the full (i.e., unpruned) parse forest
+  bool Parse(const Lattice& input,
+             Hypergraph* forest) const;
+
+ private:
+  const std::string goal_sym_;
+  const std::vector<GrammarPtr> grammars_;
+};
+
+#endif
diff --git a/decoder/cdec.cc b/decoder/cdec.cc
new file mode 100644
index 00000000..dbf32cb3
--- /dev/null
+++ b/decoder/cdec.cc
@@ -0,0 +1,592 @@
+#include <iostream>
+#include <fstream>
+#include <tr1/unordered_map>
+#include <tr1/unordered_set>
+
+#include <boost/shared_ptr.hpp>
+#include <boost/program_options.hpp>
+#include <boost/program_options/variables_map.hpp>
+
+#include "timing_stats.h"
+#include "translator.h"
+#include "phrasebased_translator.h"
+#include "aligner.h"
+#include "stringlib.h"
+#include "forest_writer.h"
+#include "hg_io.h"
+#include "filelib.h"
+#include "sampler.h"
+#include "sparse_vector.h"
+#include "tagger.h"
+#include "lextrans.h"
+#include "lexalign.h"
+#include "csplit.h"
+#include "weights.h"
+#include "tdict.h"
+#include "ff.h"
+#include "ff_factory.h"
+#include "hg_intersect.h"
+#include "apply_models.h"
+#include "viterbi.h"
+#include "kbest.h"
+#include "inside_outside.h"
+#include "exp_semiring.h"
+#include "sentence_metadata.h"
+
+using namespace std;
+using namespace std::tr1;
+using boost::shared_ptr;
+namespace po = boost::program_options;
+
+// some globals ...
+boost::shared_ptr<RandomNumberGenerator<boost::mt19937> > rng;
+static const double kMINUS_EPSILON = -1e-6;  // don't be too strict
+
+namespace Hack { void MaxTrans(const Hypergraph& in, int beam_size); }
+namespace NgramCache { void Clear(); }
+
+void ShowBanner() {
+  cerr << "cdec v1.0 (c) 2009-2010 by Chris Dyer\n";
+}
+
+void ConvertSV(const SparseVector<prob_t>& src, SparseVector<double>* trg) {
+  for (SparseVector<prob_t>::const_iterator it = src.begin(); it != src.end(); ++it)
+    trg->set_value(it->first, it->second);
+}
+
+void InitCommandLine(int argc, char** argv, po::variables_map* conf) {
+  po::options_description opts("Configuration options");
+  opts.add_options()
+        ("formalism,f",po::value<string>(),"Decoding formalism; values include SCFG, FST, PB, LexTrans (lexical translation model, also disc training), CSplit (compound splitting), Tagger (sequence labeling), LexAlign (alignment only, or EM training)")
+        ("input,i",po::value<string>()->default_value("-"),"Source file")
+        ("grammar,g",po::value<vector<string> >()->composing(),"Either SCFG grammar file(s) or phrase tables file(s)")
+        ("weights,w",po::value<string>(),"Feature weights file")
+        ("no_freeze_feature_set,Z", "Do not freeze feature set after reading feature weights file")
+        ("feature_function,F",po::value<vector<string> >()->composing(), "Additional feature function(s) (-L for list)")
+        ("list_feature_functions,L","List available feature functions")
+        ("add_pass_through_rules,P","Add rules to translate OOV words as themselves")
+	("k_best,k",po::value<int>(),"Extract the k best derivations")
+	("unique_k_best,r", "Unique k-best translation list")
+        ("aligner,a", "Run as a word/phrase aligner (src & ref required)")
+        ("intersection_strategy,I",po::value<string>()->default_value("cube_pruning"), "Intersection strategy for incorporating finite-state features; values include Cube_pruning, Full")
+        ("cubepruning_pop_limit,K",po::value<int>()->default_value(200), "Max number of pops from the candidate heap at each node")
+        ("goal",po::value<string>()->default_value("S"),"Goal symbol (SCFG & FST)")
+        ("scfg_extra_glue_grammar", po::value<string>(), "Extra glue grammar file (Glue grammars apply when i=0 but have no other span restrictions)")
+        ("scfg_no_hiero_glue_grammar,n", "No Hiero glue grammar (nb. by default the SCFG decoder adds Hiero glue rules)")
+        ("scfg_default_nt,d",po::value<string>()->default_value("X"),"Default non-terminal symbol in SCFG")
+        ("scfg_max_span_limit,S",po::value<int>()->default_value(10),"Maximum non-terminal span limit (except \"glue\" grammar)")
+	("show_tree_structure", "Show the Viterbi derivation structure")
+        ("show_expected_length", "Show the expected translation length under the model")
+        ("show_partition,z", "Compute and show the partition (inside score)")
+        ("show_cfg_search_space", "Show the search space as a CFG")
+        ("beam_prune", po::value<double>(), "Prune paths from +LM forest")
+        ("lexalign_use_null", "Support source-side null words in lexical translation")
+        ("tagger_tagset,t", po::value<string>(), "(Tagger) file containing tag set")
+        ("csplit_output_plf", "(Compound splitter) Output lattice in PLF format")
+        ("csplit_preserve_full_word", "(Compound splitter) Always include the unsegmented form in the output lattice")
+        ("extract_rules", po::value<string>(), "Extract the rules used in translation (de-duped) to this file")
+        ("graphviz","Show (constrained) translation forest in GraphViz format")
+        ("max_translation_beam,x", po::value<int>(), "Beam approximation to get max translation from the chart")
+        ("max_translation_sample,X", po::value<int>(), "Sample the max translation from the chart")
+        ("pb_max_distortion,D", po::value<int>()->default_value(4), "Phrase-based decoder: maximum distortion")
+        ("cll_gradient,G","Compute conditional log-likelihood gradient and write to STDOUT (src & ref required)")
+        ("crf_uniform_empirical", "If there are multple references use (i.e., lattice) a uniform distribution rather than posterior weighting a la EM")
+        ("feature_expectations","Write feature expectations for all features in chart (**OBJ** will be the partition)")
+        ("vector_format",po::value<string>()->default_value("b64"), "Sparse vector serialization format for feature expectations or gradients, includes (text or b64)")
+        ("combine_size,C",po::value<int>()->default_value(1), "When option -G is used, process this many sentence pairs before writing the gradient (1=emit after every sentence pair)")
+        ("forest_output,O",po::value<string>(),"Directory to write forests to")
+        ("minimal_forests,m","Write minimal forests (excludes Rule information). Such forests can be used for ML/MAP training, but not rescoring, etc.");
+  po::options_description clo("Command line options");
+  clo.add_options()
+        ("config,c", po::value<string>(), "Configuration file")
+        ("help,h", "Print this help message and exit");
+  po::options_description dconfig_options, dcmdline_options;
+  dconfig_options.add(opts);
+  dcmdline_options.add(opts).add(clo);
+
+  po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
+  if (conf->count("config")) {
+    const string cfg = (*conf)["config"].as<string>();
+    cerr << "Configuration file: " << cfg << endl;
+    ifstream config(cfg.c_str());
+    po::store(po::parse_config_file(config, dconfig_options), *conf);
+  }
+  po::notify(*conf);
+
+  if (conf->count("list_feature_functions")) {
+    cerr << "Available feature functions (specify with -F):\n";
+    global_ff_registry->DisplayList();
+    cerr << endl;
+    exit(1);
+  }
+
+  if (conf->count("help") || conf->count("formalism") == 0) {
+    cerr << dcmdline_options << endl;
+    exit(1);
+  }
+
+  const string formalism = LowercaseString((*conf)["formalism"].as<string>());
+  if (formalism != "scfg" && formalism != "fst" && formalism != "lextrans" && formalism != "pb" && formalism != "csplit" && formalism != "tagger" && formalism != "lexalign") {
+    cerr << "Error: --formalism takes only 'scfg', 'fst', 'pb', 'csplit', 'lextrans', 'lexalign', or 'tagger'\n";
+    cerr << dcmdline_options << endl;
+    exit(1);
+  }
+}
+
+// TODO move out of cdec into some sampling decoder file
+void SampleRecurse(const Hypergraph& hg, const vector<SampleSet>& ss, int n, vector<WordID>* out) {
+  const SampleSet& s = ss[n];
+  int i = rng->SelectSample(s);
+  const Hypergraph::Edge& edge = hg.edges_[hg.nodes_[n].in_edges_[i]];
+  vector<vector<WordID> > ants(edge.tail_nodes_.size());
+  for (int j = 0; j < ants.size(); ++j)
+    SampleRecurse(hg, ss, edge.tail_nodes_[j], &ants[j]);
+
+  vector<const vector<WordID>*> pants(ants.size());
+  for (int j = 0; j < ants.size(); ++j) pants[j] = &ants[j];
+  edge.rule_->ESubstitute(pants, out);
+}
+
+struct SampleSort {
+  bool operator()(const pair<int,string>& a, const pair<int,string>& b) const {
+    return a.first > b.first;
+  }
+};
+
+// TODO move out of cdec into some sampling decoder file
+void MaxTranslationSample(Hypergraph* hg, const int samples, const int k) {
+  unordered_map<string, int, boost::hash<string> > m;
+  hg->PushWeightsToGoal();
+  const int num_nodes = hg->nodes_.size();
+  vector<SampleSet> ss(num_nodes);
+  for (int i = 0; i < num_nodes; ++i) {
+    SampleSet& s = ss[i];
+    const vector<int>& in_edges = hg->nodes_[i].in_edges_;
+    for (int j = 0; j < in_edges.size(); ++j) {
+      s.add(hg->edges_[in_edges[j]].edge_prob_);
+    }
+  }
+  for (int i = 0; i < samples; ++i) {
+    vector<WordID> yield;
+    SampleRecurse(*hg, ss, hg->nodes_.size() - 1, &yield);
+    const string trans = TD::GetString(yield);
+    ++m[trans];
+  }
+  vector<pair<int, string> > dist;
+  for (unordered_map<string, int, boost::hash<string> >::iterator i = m.begin();
+         i != m.end(); ++i) {
+    dist.push_back(make_pair(i->second, i->first));
+  }
+  sort(dist.begin(), dist.end(), SampleSort());
+  if (k) {
+    for (int i = 0; i < k; ++i)
+      cout << dist[i].first << " ||| " << dist[i].second << endl;
+  } else {
+    cout << dist[0].second << endl;
+  }
+}
+
+// TODO decoder output should probably be moved to another file
+void DumpKBest(const int sent_id, const Hypergraph& forest, const int k, const bool unique) {
+cerr << "In kbest\n"; 
+ if (unique) {
+    KBest::KBestDerivations<vector<WordID>, ESentenceTraversal, KBest::FilterUnique> kbest(forest, k);
+    for (int i = 0; i < k; ++i) {
+      const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal, KBest::FilterUnique>::Derivation* d =
+        kbest.LazyKthBest(forest.nodes_.size() - 1, i);
+      if (!d) break;
+      cout << sent_id << " ||| " << TD::GetString(d->yield) << " ||| "
+           << d->feature_values << " ||| " << log(d->score) << endl;
+    }
+  } else {
+    KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(forest, k);
+    for (int i = 0; i < k; ++i) {
+      const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
+        kbest.LazyKthBest(forest.nodes_.size() - 1, i);
+      if (!d) break;
+      cout << sent_id << " ||| " << TD::GetString(d->yield) << " ||| "
+           << d->feature_values << " ||| " << log(d->score) << endl;
+    }
+  }
+}
+
+struct ELengthWeightFunction {
+  double operator()(const Hypergraph::Edge& e) const {
+    return e.rule_->ELength() - e.rule_->Arity();
+  }
+};
+
+
+struct TRPHash {
+  size_t operator()(const TRulePtr& o) const { return reinterpret_cast<size_t>(o.get()); }
+};
+static void ExtractRulesDedupe(const Hypergraph& hg, ostream* os) {
+  static unordered_set<TRulePtr, TRPHash> written;
+  for (int i = 0; i < hg.edges_.size(); ++i) {
+    const TRulePtr& rule = hg.edges_[i].rule_;
+    if (written.insert(rule).second) {
+      (*os) << rule->AsString() << endl;
+    }
+  }
+}
+
+void register_feature_functions();
+
+int main(int argc, char** argv) {
+  global_ff_registry.reset(new FFRegistry);
+  register_feature_functions();
+  ShowBanner();
+  po::variables_map conf;
+  InitCommandLine(argc, argv, &conf);
+  const bool write_gradient = conf.count("cll_gradient");
+  const bool feature_expectations = conf.count("feature_expectations");
+  if (write_gradient && feature_expectations) {
+    cerr << "You can only specify --gradient or --feature_expectations, not both!\n";
+    exit(1);
+  }
+  const bool output_training_vector = (write_gradient || feature_expectations);
+
+  boost::shared_ptr<Translator> translator;
+  const string formalism = LowercaseString(conf["formalism"].as<string>());
+  const bool csplit_preserve_full_word = conf.count("csplit_preserve_full_word");
+  if (csplit_preserve_full_word &&
+      (formalism != "csplit" || !conf.count("beam_prune"))) {
+    cerr << "--csplit_preserve_full_word should only be "
+         << "used with csplit AND --beam_prune!\n";
+    exit(1);
+  }
+  const bool csplit_output_plf = conf.count("csplit_output_plf");
+  if (csplit_output_plf && formalism != "csplit") {
+    cerr << "--csplit_output_plf should only be used with csplit!\n";
+    exit(1);
+  }
+
+  // load feature weights (and possibly freeze feature set)
+  vector<double> feature_weights;
+  Weights w;
+  if (conf.count("weights")) {
+    w.InitFromFile(conf["weights"].as<string>());
+    feature_weights.resize(FD::NumFeats());
+    w.InitVector(&feature_weights);
+    if (!conf.count("no_freeze_feature_set")) {
+      cerr << "Freezing feature set (use --no_freeze_feature_set to change)." << endl;
+      FD::Freeze();
+    }
+  }
+
+  // set up translation back end
+  if (formalism == "scfg")
+    translator.reset(new SCFGTranslator(conf));
+  else if (formalism == "fst")
+    translator.reset(new FSTTranslator(conf));
+  else if (formalism == "pb")
+    translator.reset(new PhraseBasedTranslator(conf));
+  else if (formalism == "csplit")
+    translator.reset(new CompoundSplit(conf));
+  else if (formalism == "lextrans")
+    translator.reset(new LexicalTrans(conf));
+  else if (formalism == "lexalign")
+    translator.reset(new LexicalAlign(conf));
+  else if (formalism == "tagger")
+    translator.reset(new Tagger(conf));
+  else
+    assert(!"error");
+
+  // set up additional scoring features
+  vector<shared_ptr<FeatureFunction> > pffs;
+  vector<const FeatureFunction*> late_ffs;
+  if (conf.count("feature_function") > 0) {
+    const vector<string>& add_ffs = conf["feature_function"].as<vector<string> >();
+    for (int i = 0; i < add_ffs.size(); ++i) {
+      string ff, param;
+      SplitCommandAndParam(add_ffs[i], &ff, &param);
+      cerr << "Feature: " << ff;
+      if (param.size() > 0) cerr << " (with config parameters '" << param << "')\n";
+      else cerr << " (no config parameters)\n";
+      shared_ptr<FeatureFunction> pff = global_ff_registry->Create(ff, param);
+      if (!pff) { exit(1); }
+      // TODO check that multiple features aren't trying to set the same fid
+      pffs.push_back(pff);
+      late_ffs.push_back(pff.get());
+    }
+  }
+  ModelSet late_models(feature_weights, late_ffs);
+  int palg = 1;
+  if (LowercaseString(conf["intersection_strategy"].as<string>()) == "full") {
+    palg = 0;
+    cerr << "Using full intersection (no pruning).\n";
+  }
+  const IntersectionConfiguration inter_conf(palg, conf["cubepruning_pop_limit"].as<int>());
+
+  const int sample_max_trans = conf.count("max_translation_sample") ?
+    conf["max_translation_sample"].as<int>() : 0;
+  if (sample_max_trans)
+    rng.reset(new RandomNumberGenerator<boost::mt19937>);
+  const bool aligner_mode = conf.count("aligner");
+  const bool minimal_forests = conf.count("minimal_forests");
+  const bool graphviz = conf.count("graphviz");
+  const bool encode_b64 = conf["vector_format"].as<string>() == "b64";
+  const bool kbest = conf.count("k_best");
+  const bool unique_kbest = conf.count("unique_k_best");
+  const bool crf_uniform_empirical = conf.count("crf_uniform_empirical");
+  shared_ptr<WriteFile> extract_file;
+  if (conf.count("extract_rules"))
+    extract_file.reset(new WriteFile(conf["extract_rules"].as<string>()));
+
+  int combine_size = conf["combine_size"].as<int>();
+  if (combine_size < 1) combine_size = 1;
+  const string input = conf["input"].as<string>();
+  cerr << "Reading input from " << ((input == "-") ? "STDIN" : input.c_str()) << endl;
+  ReadFile in_read(input);
+  istream *in = in_read.stream();
+  assert(*in);
+
+  SparseVector<prob_t> acc_vec;  // accumulate gradient
+  double acc_obj = 0; // accumulate objective
+  int g_count = 0;    // number of gradient pieces computed
+  int sent_id = -1;         // line counter
+
+  while(*in) {
+    NgramCache::Clear();   // clear ngram cache for remote LM (if used)
+    Timer::Summarize();
+    ++sent_id;
+    string buf;
+    getline(*in, buf);
+    if (buf.empty()) continue;
+    map<string, string> sgml;
+    ProcessAndStripSGML(&buf, &sgml);
+    if (sgml.find("id") != sgml.end())
+      sent_id = atoi(sgml["id"].c_str());
+
+    cerr << "\nINPUT: ";
+    if (buf.size() < 100)
+      cerr << buf << endl;
+    else {
+     size_t x = buf.rfind(" ", 100);
+     if (x == string::npos) x = 100;
+     cerr << buf.substr(0, x) << " ..." << endl;
+    }
+    cerr << "  id = " << sent_id << endl;
+    string to_translate;
+    Lattice ref;
+    ParseTranslatorInputLattice(buf, &to_translate, &ref);
+    const bool has_ref = ref.size() > 0;
+    SentenceMetadata smeta(sent_id, ref);
+    const bool hadoop_counters = (write_gradient);
+    Hypergraph forest;          // -LM forest
+    translator->ProcessMarkupHints(sgml);
+    Timer t("Translation");
+    const bool translation_successful =
+      translator->Translate(to_translate, &smeta, feature_weights, &forest);
+    translator->SentenceComplete();
+    if (!translation_successful) {
+      cerr << "  NO PARSE FOUND.\n";
+      if (hadoop_counters)
+        cerr << "reporter:counter:UserCounters,FParseFailed,1" << endl;
+      cout << endl << flush;
+      continue;
+    }
+    cerr << "  -LM forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
+    cerr << "  -LM forest       (paths): " << forest.NumberOfPaths() << endl;
+    if (conf.count("show_expected_length")) {
+      const PRPair<double, double> res =
+        Inside<PRPair<double, double>,
+               PRWeightFunction<double, EdgeProb, double, ELengthWeightFunction> >(forest);
+      cerr << "  Expected length  (words): " << res.r / res.p << "\t" << res << endl;
+    }
+    if (conf.count("show_partition")) {
+      const prob_t z = Inside<prob_t, EdgeProb>(forest);
+      cerr << "  -LM partition     log(Z): " << log(z) << endl;
+    }
+    if (extract_file)
+      ExtractRulesDedupe(forest, extract_file->stream());
+    vector<WordID> trans;
+    const prob_t vs = ViterbiESentence(forest, &trans);
+    cerr << "  -LM Viterbi: " << TD::GetString(trans) << endl;
+    if (conf.count("show_tree_structure"))
+      cerr << "  -LM    tree: " << ViterbiETree(forest) << endl;;
+    cerr << "  -LM Viterbi: " << log(vs) << endl;
+
+    bool has_late_models = !late_models.empty();
+    if (has_late_models) {
+      forest.Reweight(feature_weights);
+      forest.SortInEdgesByEdgeWeights();
+      Hypergraph lm_forest;
+      ApplyModelSet(forest,
+                    smeta,
+                    late_models,
+                    inter_conf,
+                    &lm_forest);
+      forest.swap(lm_forest);
+      forest.Reweight(feature_weights);
+      trans.clear();
+      ViterbiESentence(forest, &trans);
+      cerr << "  +LM forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
+      cerr << "  +LM forest       (paths): " << forest.NumberOfPaths() << endl;
+      cerr << "  +LM Viterbi: " << TD::GetString(trans) << endl;
+    }
+    if (conf.count("beam_prune")) {
+      vector<bool> preserve_mask(forest.edges_.size(), false);
+      if (csplit_preserve_full_word)
+        preserve_mask[CompoundSplit::GetFullWordEdgeIndex(forest)] = true;
+      forest.BeamPruneInsideOutside(1.0, false, conf["beam_prune"].as<double>(), &preserve_mask);
+      cerr << "  Pruned forest    (paths): " << forest.NumberOfPaths() << endl;
+    }
+
+    if (conf.count("forest_output") && !has_ref) {
+      ForestWriter writer(conf["forest_output"].as<string>(), sent_id);
+      if (FileExists(writer.fname_)) {
+        cerr << "  Unioning...\n";
+        Hypergraph new_hg;
+        {
+          ReadFile rf(writer.fname_);
+          bool succeeded = HypergraphIO::ReadFromJSON(rf.stream(), &new_hg);
+          assert(succeeded);
+        }
+        new_hg.Union(forest);
+        bool succeeded = writer.Write(new_hg, minimal_forests);
+        assert(succeeded);
+      } else {
+        bool succeeded = writer.Write(forest, minimal_forests);
+        assert(succeeded);
+      }
+    }
+
+    if (sample_max_trans) {
+      MaxTranslationSample(&forest, sample_max_trans, conf.count("k_best") ? conf["k_best"].as<int>() : 0);
+    } else {
+      if (kbest) {
+        DumpKBest(sent_id, forest, conf["k_best"].as<int>(), unique_kbest);
+      } else if (csplit_output_plf) {
+        cout << HypergraphIO::AsPLF(forest, false) << endl;
+      } else {
+        if (!graphviz && !has_ref) {
+          cout << TD::GetString(trans) << endl << flush;
+        }
+      }
+    }
+
+    const int max_trans_beam_size = conf.count("max_translation_beam") ?
+      conf["max_translation_beam"].as<int>() : 0;
+    if (max_trans_beam_size) {
+      Hack::MaxTrans(forest, max_trans_beam_size);
+      continue;
+    }
+
+    if (graphviz && !has_ref) forest.PrintGraphviz();
+
+    // the following are only used if write_gradient is true!
+    SparseVector<prob_t> full_exp, ref_exp, gradient;
+    double log_z = 0, log_ref_z = 0;
+    if (write_gradient) {
+      const prob_t z = InsideOutside<prob_t, EdgeProb, SparseVector<prob_t>, EdgeFeaturesAndProbWeightFunction>(forest, &full_exp);
+      log_z = log(z);
+      full_exp /= z;
+    }
+    if (conf.count("show_cfg_search_space"))
+      HypergraphIO::WriteAsCFG(forest);
+    if (has_ref) {
+      if (HG::Intersect(ref, &forest)) {
+        cerr << "  Constr. forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
+        cerr << "  Constr. forest       (paths): " << forest.NumberOfPaths() << endl;
+        if (crf_uniform_empirical) {
+          cerr << "  USING UNIFORM WEIGHTS\n";
+          for (int i = 0; i < forest.edges_.size(); ++i)
+            forest.edges_[i].edge_prob_=prob_t::One();
+        } else {
+          forest.Reweight(feature_weights);
+          cerr << "  Constr. VitTree: " << ViterbiFTree(forest) << endl;
+        }
+	if (hadoop_counters)
+          cerr << "reporter:counter:UserCounters,SentencePairsParsed,1" << endl;
+        if (conf.count("show_partition")) {
+           const prob_t z = Inside<prob_t, EdgeProb>(forest);
+           cerr << "  Contst. partition  log(Z): " << log(z) << endl;
+        }
+        //DumpKBest(sent_id, forest, 1000);
+        if (conf.count("forest_output")) {
+          ForestWriter writer(conf["forest_output"].as<string>(), sent_id);
+          if (FileExists(writer.fname_)) {
+            cerr << "  Unioning...\n";
+            Hypergraph new_hg;
+            {
+              ReadFile rf(writer.fname_);
+              bool succeeded = HypergraphIO::ReadFromJSON(rf.stream(), &new_hg);
+              assert(succeeded);
+            }
+            new_hg.Union(forest);
+            bool succeeded = writer.Write(new_hg, minimal_forests);
+            assert(succeeded);
+          } else {
+            bool succeeded = writer.Write(forest, minimal_forests);
+            assert(succeeded);
+          }
+        }
+        if (aligner_mode && !output_training_vector)
+          AlignerTools::WriteAlignment(smeta.GetSourceLattice(), smeta.GetReference(), forest, &cout);
+        if (write_gradient) {
+          const prob_t ref_z = InsideOutside<prob_t, EdgeProb, SparseVector<prob_t>, EdgeFeaturesAndProbWeightFunction>(forest, &ref_exp);
+          ref_exp /= ref_z;
+          if (crf_uniform_empirical) {
+            log_ref_z = ref_exp.dot(feature_weights);
+          } else {
+            log_ref_z = log(ref_z);
+          }
+          //cerr << "      MODEL LOG Z: " << log_z << endl;
+          //cerr << "  EMPIRICAL LOG Z: " << log_ref_z << endl;
+          if ((log_z - log_ref_z) < kMINUS_EPSILON) {
+            cerr << "DIFF. ERR! log_z < log_ref_z: " << log_z << " " << log_ref_z << endl;
+            exit(1);
+          }
+          assert(!isnan(log_ref_z));
+          ref_exp -= full_exp;
+          acc_vec += ref_exp;
+          acc_obj += (log_z - log_ref_z);
+        }
+        if (feature_expectations) {
+          const prob_t z = 
+            InsideOutside<prob_t, EdgeProb, SparseVector<prob_t>, EdgeFeaturesAndProbWeightFunction>(forest, &ref_exp);
+          ref_exp /= z;
+          acc_obj += log(z);
+          acc_vec += ref_exp;
+        }
+
+        if (output_training_vector) {
+          acc_vec.clear_value(0);
+          ++g_count;
+          if (g_count % combine_size == 0) {
+            if (encode_b64) {
+              cout << "0\t";
+              SparseVector<double> dav; ConvertSV(acc_vec, &dav);
+              B64::Encode(acc_obj, dav, &cout);
+              cout << endl << flush;
+            } else {
+              cout << "0\t**OBJ**=" << acc_obj << ';' <<  acc_vec << endl << flush;
+            }
+            acc_vec.clear();
+            acc_obj = 0;
+          }
+        }
+        if (conf.count("graphviz")) forest.PrintGraphviz();
+      } else {
+        cerr << "  REFERENCE UNREACHABLE.\n";
+        if (write_gradient) {
+	  if (hadoop_counters)
+            cerr << "reporter:counter:UserCounters,EFParseFailed,1" << endl;
+          cout << endl << flush;
+	}
+      }
+    }
+  }
+  if (output_training_vector && !acc_vec.empty()) {
+    if (encode_b64) {
+      cout << "0\t";
+      SparseVector<double> dav; ConvertSV(acc_vec, &dav);
+      B64::Encode(acc_obj, dav, &cout);
+      cout << endl << flush;
+    } else {
+      cout << "0\t**OBJ**=" << acc_obj << ';' << acc_vec << endl << flush;
+    }
+  }
+}
+
diff --git a/decoder/cdec_ff.cc b/decoder/cdec_ff.cc
new file mode 100644
index 00000000..d0b93795
--- /dev/null
+++ b/decoder/cdec_ff.cc
@@ -0,0 +1,32 @@
+#include <boost/shared_ptr.hpp>
+
+#include "ff.h"
+#include "ff_lm.h"
+#include "ff_csplit.h"
+#include "ff_wordalign.h"
+#include "ff_tagger.h"
+#include "ff_factory.h"
+
+boost::shared_ptr<FFRegistry> global_ff_registry;
+
+void register_feature_functions() {
+  global_ff_registry->Register("LanguageModel", new FFFactory<LanguageModel>);
+#ifdef HAVE_RANDLM
+  global_ff_registry->Register("RandLM", new FFFactory<LanguageModelRandLM>);
+#endif
+  global_ff_registry->Register("WordPenalty", new FFFactory<WordPenalty>);
+  global_ff_registry->Register("SourceWordPenalty", new FFFactory<SourceWordPenalty>);
+  global_ff_registry->Register("ArityPenalty", new FFFactory<ArityPenalty>);
+  global_ff_registry->Register("RelativeSentencePosition", new FFFactory<RelativeSentencePosition>);
+  global_ff_registry->Register("Model2BinaryFeatures", new FFFactory<Model2BinaryFeatures>);
+  global_ff_registry->Register("MarkovJump", new FFFactory<MarkovJump>);
+  global_ff_registry->Register("MarkovJumpFClass", new FFFactory<MarkovJumpFClass>);
+  global_ff_registry->Register("SourcePOSBigram", new FFFactory<SourcePOSBigram>);
+  global_ff_registry->Register("BlunsomSynchronousParseHack", new FFFactory<BlunsomSynchronousParseHack>);
+  global_ff_registry->Register("AlignerResults", new FFFactory<AlignerResults>);
+  global_ff_registry->Register("CSplit_BasicFeatures", new FFFactory<BasicCSplitFeatures>);
+  global_ff_registry->Register("CSplit_ReverseCharLM", new FFFactory<ReverseCharLMCSplitFeature>);
+  global_ff_registry->Register("Tagger_BigramIdentity", new FFFactory<Tagger_BigramIdentity>);
+  global_ff_registry->Register("LexicalPairIdentity", new FFFactory<LexicalPairIdentity>);
+};
+
diff --git a/decoder/csplit.cc b/decoder/csplit.cc
new file mode 100644
index 00000000..b1a30fb0
--- /dev/null
+++ b/decoder/csplit.cc
@@ -0,0 +1,173 @@
+#include "csplit.h"
+
+#include <iostream>
+
+#include "filelib.h"
+#include "stringlib.h"
+#include "hg.h"
+#include "tdict.h"
+#include "grammar.h"
+#include "sentence_metadata.h"
+
+using namespace std;
+
+struct CompoundSplitImpl {
+  CompoundSplitImpl(const boost::program_options::variables_map& conf) :
+      fugen_elements_(true),   // TODO configure
+      min_size_(3),
+      kXCAT(TD::Convert("X")*-1),
+      kWORDBREAK_RULE(new TRule("[X] ||| # ||| #")),
+      kTEMPLATE_RULE(new TRule("[X] ||| [X,1] ? ||| [1] ?")),
+      kGOAL_RULE(new TRule("[Goal] ||| [X,1] ||| [1]")),
+      kFUGEN_S(FD::Convert("FugS")),
+      kFUGEN_N(FD::Convert("FugN")) {}
+
+  void PasteTogetherStrings(const vector<string>& chars,
+                            const int i,
+                            const int j,
+                            string* yield) {
+    int size = 0;
+    for (int k=i; k<j; ++k)
+      size += chars[k].size();
+    yield->resize(size);
+    int cur = 0;
+    for (int k=i; k<j; ++k) {
+      const string& cs = chars[k];
+      for (int l = 0; l < cs.size(); ++l)
+        (*yield)[cur++] = cs[l];
+    }
+  }
+
+  void BuildTrellis(const vector<string>& chars,
+                    Hypergraph* forest) {
+    vector<int> nodes(chars.size()+1, -1);
+    nodes[0] = forest->AddNode(kXCAT)->id_;       // source
+    const int left_rule = forest->AddEdge(kWORDBREAK_RULE, Hypergraph::TailNodeVector())->id_;
+    forest->ConnectEdgeToHeadNode(left_rule, nodes[0]);
+
+    const int max_split_ = max(static_cast<int>(chars.size()) - min_size_ + 1, 1);
+    cerr << "max: " << max_split_ << "  " << " min: " << min_size_ << endl;
+    for (int i = min_size_; i < max_split_; ++i)
+      nodes[i] = forest->AddNode(kXCAT)->id_;
+    assert(nodes.back() == -1);
+    nodes.back() = forest->AddNode(kXCAT)->id_;   // sink
+
+    for (int i = 0; i < max_split_; ++i) {
+      if (nodes[i] < 0) continue;
+      const int start = min(i + min_size_, static_cast<int>(chars.size()));
+      for (int j = start; j <= chars.size(); ++j) {
+        if (nodes[j] < 0) continue;
+        string yield;
+        PasteTogetherStrings(chars, i, j, &yield);
+        // cerr << "[" << i << "," << j << "] " << yield << endl;
+        TRulePtr rule = TRulePtr(new TRule(*kTEMPLATE_RULE));
+        rule->e_[1] = rule->f_[1] = TD::Convert(yield);
+        // cerr << rule->AsString() << endl;
+        int edge = forest->AddEdge(
+          rule,
+          Hypergraph::TailNodeVector(1, nodes[i]))->id_;
+        forest->ConnectEdgeToHeadNode(edge, nodes[j]);
+        forest->edges_[edge].i_ = i;
+        forest->edges_[edge].j_ = j;
+
+        // handle "fugenelemente" here
+        // don't delete "fugenelemente" at the end of words
+        if (fugen_elements_ && j != chars.size()) {
+          const int len = yield.size();
+          string alt;
+          int fid = 0;
+          if (len > (min_size_ + 2) && yield[len-1] == 's' && yield[len-2] == 'e') {
+            alt = yield.substr(0, len - 2);
+            fid = kFUGEN_S;
+          } else if (len > (min_size_ + 1) && yield[len-1] == 's') {
+            alt = yield.substr(0, len - 1);
+            fid = kFUGEN_S;
+          } else if (len > (min_size_ + 2) && yield[len-2] == 'e' && yield[len-1] == 'n') {
+            alt = yield.substr(0, len - 1);
+            fid = kFUGEN_N;
+          }
+          if (alt.size()) {
+            TRulePtr altrule = TRulePtr(new TRule(*rule));
+            altrule->e_[1] = TD::Convert(alt);
+            // cerr << altrule->AsString() << endl;
+            int edge = forest->AddEdge(
+              altrule,
+              Hypergraph::TailNodeVector(1, nodes[i]))->id_;
+            forest->ConnectEdgeToHeadNode(edge, nodes[j]);
+            forest->edges_[edge].feature_values_.set_value(fid, 1.0);
+            forest->edges_[edge].i_ = i;
+            forest->edges_[edge].j_ = j;
+          }
+        }
+      }
+    }
+
+    // add goal rule
+    Hypergraph::TailNodeVector tail(1, forest->nodes_.size() - 1);
+    Hypergraph::Node* goal = forest->AddNode(TD::Convert("Goal")*-1);
+    Hypergraph::Edge* hg_edge = forest->AddEdge(kGOAL_RULE, tail);
+    forest->ConnectEdgeToHeadNode(hg_edge, goal);
+  }
+ private:
+  const bool fugen_elements_;
+  const int min_size_;
+  const WordID kXCAT;
+  const TRulePtr kWORDBREAK_RULE;
+  const TRulePtr kTEMPLATE_RULE;
+  const TRulePtr kGOAL_RULE;
+  const int kFUGEN_S;
+  const int kFUGEN_N;
+};
+
+CompoundSplit::CompoundSplit(const boost::program_options::variables_map& conf) :
+  pimpl_(new CompoundSplitImpl(conf)) {}
+
+static void SplitUTF8String(const string& in, vector<string>* out) {
+  out->resize(in.size());
+  int i = 0;
+  int c = 0;
+  while (i < in.size()) {
+    const int len = UTF8Len(in[i]);
+    assert(len);
+    (*out)[c] = in.substr(i, len);
+    ++c;
+    i += len;
+  }
+  out->resize(c);
+}
+
+bool CompoundSplit::TranslateImpl(const string& input,
+                      SentenceMetadata* smeta,
+                      const vector<double>& weights,
+                      Hypergraph* forest) {
+  if (input.find(" ") != string::npos) {
+    cerr << "  BAD INPUT: " << input << "\n    CompoundSplit expects single words\n";
+    abort();
+  }
+  vector<string> in;
+  SplitUTF8String(input, &in);
+  smeta->SetSourceLength(in.size());  // TODO do utf8 or somethign
+  for (int i = 0; i < in.size(); ++i)
+    smeta->src_lattice_.push_back(vector<LatticeArc>(1, LatticeArc(TD::Convert(in[i]), 0.0, 1)));
+  pimpl_->BuildTrellis(in, forest);
+  forest->Reweight(weights);
+  return true;
+}
+
+int CompoundSplit::GetFullWordEdgeIndex(const Hypergraph& forest) {
+  assert(forest.nodes_.size() > 0);
+  const vector<int> out_edges = forest.nodes_[0].out_edges_;
+  int max_edge = -1;
+  int max_j = -1;
+  for (int i = 0; i < out_edges.size(); ++i) {
+    const int j = forest.edges_[out_edges[i]].j_;
+    if (j > max_j) {
+      max_j = j;
+      max_edge = out_edges[i];
+    }
+  }
+  assert(max_edge >= 0);
+  assert(max_edge < forest.edges_.size());
+  return max_edge;
+}
+
diff --git a/decoder/csplit.h b/decoder/csplit.h
new file mode 100644
index 00000000..82ed23fc
--- /dev/null
+++ b/decoder/csplit.h
@@ -0,0 +1,30 @@
+#ifndef _CSPLIT_H_
+#define _CSPLIT_H_
+
+#include "translator.h"
+#include "lattice.h"
+
+// this "translator" takes single words (with NO SPACES) and segments
+// them using the approach described in:
+//
+// C. Dyer. (2009) Using a maximum entropy model to build segmentation
+//                 lattices for MT. In Proceedings of NAACL HLT 2009.
+// note, an extra word space marker # is inserted at the left edge of
+// the forest!
+struct CompoundSplitImpl;
+struct CompoundSplit : public Translator {
+  CompoundSplit(const boost::program_options::variables_map& conf);
+  bool TranslateImpl(const std::string& input,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* forest);
+
+  // given a forest generated by CompoundSplit::Translate,
+  // find the edge representing the unsegmented form
+  static int GetFullWordEdgeIndex(const Hypergraph& forest);
+
+ private:
+  boost::shared_ptr<CompoundSplitImpl> pimpl_;
+};
+
+#endif
diff --git a/decoder/dict.h b/decoder/dict.h
new file mode 100644
index 00000000..72e82e6d
--- /dev/null
+++ b/decoder/dict.h
@@ -0,0 +1,43 @@
+#ifndef DICT_H_
+#define DICT_H_
+
+#include <cassert>
+#include <cstring>
+#include <tr1/unordered_map>
+#include <string>
+#include <vector>
+
+#include <boost/functional/hash.hpp>
+
+#include "wordid.h"
+
+class Dict {
+ typedef std::tr1::unordered_map<std::string, WordID, boost::hash<std::string> > Map;
+ public:
+  Dict() : b0_("<bad0>") { words_.reserve(1000); }
+  inline int max() const { return words_.size(); }
+  inline WordID Convert(const std::string& word, bool frozen = false) {
+    Map::iterator i = d_.find(word);
+    if (i == d_.end()) {
+      if (frozen)
+        return 0;
+      words_.push_back(word);
+      d_[word] = words_.size();
+      return words_.size();
+    } else {
+      return i->second;
+    }
+  }
+  inline const std::string& Convert(const WordID& id) const {
+    if (id == 0) return b0_;
+    assert(id <= words_.size());
+    return words_[id-1];
+  }
+  void clear() { words_.clear(); d_.clear(); }
+ private:
+  const std::string b0_;
+  std::vector<std::string> words_;
+  Map d_;
+};
+
+#endif
diff --git a/decoder/dict_test.cc b/decoder/dict_test.cc
new file mode 100644
index 00000000..694877fa
--- /dev/null
+++ b/decoder/dict_test.cc
@@ -0,0 +1,50 @@
+#include "dict.h"
+
+#include "fdict.h"
+
+#include <iostream>
+#include <gtest/gtest.h>
+#include <cassert>
+#include "filelib.h"
+
+#include "tdict.h"
+
+using namespace std;
+
+class DTest : public testing::Test {
+ public:
+  DTest() {}
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+};
+
+TEST_F(DTest, Convert) {
+  Dict d;
+  WordID a = d.Convert("foo");
+  WordID b = d.Convert("bar");
+  std::string x = "foo";
+  WordID c = d.Convert(x);
+  EXPECT_NE(a, b);
+  EXPECT_EQ(a, c);
+  EXPECT_EQ(d.Convert(a), "foo");
+  EXPECT_EQ(d.Convert(b), "bar");
+}
+
+TEST_F(DTest, FDictTest) {
+  int fid = FD::Convert("First");
+  EXPECT_GT(fid, 0);
+  EXPECT_EQ(FD::Convert(fid), "First");
+  string x = FD::Escape("=");
+  cerr << x << endl;
+  EXPECT_NE(x, "=");
+  x = FD::Escape(";");
+  cerr << x << endl;
+  EXPECT_NE(x, ";");
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
diff --git a/decoder/earley_composer.cc b/decoder/earley_composer.cc
new file mode 100644
index 00000000..f6a01e52
--- /dev/null
+++ b/decoder/earley_composer.cc
@@ -0,0 +1,726 @@
+#include "earley_composer.h"
+
+#include <iostream>
+#include <fstream>
+#include <map>
+#include <queue>
+#include <tr1/unordered_set>
+
+#include <boost/shared_ptr.hpp>
+#include <boost/program_options.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/lexical_cast.hpp>
+
+#include "phrasetable_fst.h"
+#include "sparse_vector.h"
+#include "tdict.h"
+#include "hg.h"
+
+using boost::shared_ptr;
+namespace po = boost::program_options;
+using namespace std;
+using namespace std::tr1;
+
+// Define the following macro if you want to see lots of debugging output
+// when you run the chart parser
+#undef DEBUG_CHART_PARSER
+
+// A few constants used by the chart parser ///////////////
+static const int kMAX_NODES = 2000000;
+static const string kPHRASE_STRING = "X";
+static bool constants_need_init = true;
+static WordID kUNIQUE_START;
+static WordID kPHRASE;
+static TRulePtr kX1X2;
+static TRulePtr kX1;
+static WordID kEPS;
+static TRulePtr kEPSRule;
+
+static void InitializeConstants() {
+  if (constants_need_init) {
+    kPHRASE = TD::Convert(kPHRASE_STRING) * -1;
+    kUNIQUE_START = TD::Convert("S") * -1;
+    kX1X2.reset(new TRule("[X] ||| [X,1] [X,2] ||| [X,1] [X,2]"));
+    kX1.reset(new TRule("[X] ||| [X,1] ||| [X,1]"));
+    kEPSRule.reset(new TRule("[X] ||| <eps> ||| <eps>"));
+    kEPS = TD::Convert("<eps>");
+    constants_need_init = false;
+  }
+}
+////////////////////////////////////////////////////////////
+
+class EGrammarNode {
+  friend bool EarleyComposer::Compose(const Hypergraph& src_forest, Hypergraph* trg_forest);
+  friend void AddGrammarRule(const string& r, map<WordID, EGrammarNode>* g);
+ public:
+#ifdef DEBUG_CHART_PARSER
+  string hint;
+#endif
+  EGrammarNode() : is_some_rule_complete(false), is_root(false) {}
+  const map<WordID, EGrammarNode>& GetTerminals() const { return tptr; }
+  const map<WordID, EGrammarNode>& GetNonTerminals() const { return ntptr; }
+  bool HasNonTerminals() const { return (!ntptr.empty()); }
+  bool HasTerminals() const { return (!tptr.empty()); }
+  bool RuleCompletes() const {
+    return (is_some_rule_complete || (ntptr.empty() && tptr.empty()));
+  }
+  bool GrammarContinues() const {
+    return !(ntptr.empty() && tptr.empty());
+  }
+  bool IsRoot() const {
+    return is_root;
+  }
+  // these are the features associated with the rule from the start
+  // node up to this point.  If you use these features, you must
+  // not Extend() this rule.
+  const SparseVector<double>& GetCFGProductionFeatures() const {
+    return input_features;
+  }
+
+  const EGrammarNode* Extend(const WordID& t) const {
+    if (t < 0) {
+      map<WordID, EGrammarNode>::const_iterator it = ntptr.find(t);
+      if (it == ntptr.end()) return NULL;
+      return &it->second;
+    } else {
+      map<WordID, EGrammarNode>::const_iterator it = tptr.find(t);
+      if (it == tptr.end()) return NULL;
+      return &it->second;
+    }
+  }
+
+ private:
+  map<WordID, EGrammarNode> tptr;
+  map<WordID, EGrammarNode> ntptr;
+  SparseVector<double> input_features;
+  bool is_some_rule_complete;
+  bool is_root;
+};
+typedef map<WordID, EGrammarNode> EGrammar;    // indexed by the rule LHS
+
+// edges are immutable once created
+struct Edge {
+#ifdef DEBUG_CHART_PARSER
+  static int id_count;
+  const int id;
+#endif
+  const WordID cat;                   // lhs side of rule proved/being proved
+  const EGrammarNode* const dot;      // dot position
+  const FSTNode* const q;             // start of span
+  const FSTNode* const r;             // end of span
+  const Edge* const active_parent;    // back pointer, NULL for PREDICT items
+  const Edge* const passive_parent;   // back pointer, NULL for SCAN and PREDICT items
+  const TargetPhraseSet* const tps;   // translations
+  shared_ptr<SparseVector<double> > features; // features from CFG rule
+
+  bool IsPassive() const {
+    // when a rule is completed, this value will be set
+    return static_cast<bool>(features);
+  }
+  bool IsActive() const { return !IsPassive(); }
+  bool IsInitial() const {
+    return !(active_parent || passive_parent);
+  }
+  bool IsCreatedByScan() const {
+    return active_parent && !passive_parent && !dot->IsRoot();
+  }
+  bool IsCreatedByPredict() const {
+    return dot->IsRoot();
+  }
+  bool IsCreatedByComplete() const {
+    return active_parent && passive_parent;
+  }
+
+  // constructor for PREDICT
+  Edge(WordID c, const EGrammarNode* d, const FSTNode* q_and_r) :
+#ifdef DEBUG_CHART_PARSER
+    id(++id_count),
+#endif
+    cat(c), dot(d), q(q_and_r), r(q_and_r), active_parent(NULL), passive_parent(NULL), tps(NULL) {}
+  Edge(WordID c, const EGrammarNode* d, const FSTNode* q_and_r, const Edge* act_parent) :
+#ifdef DEBUG_CHART_PARSER
+    id(++id_count),
+#endif
+    cat(c), dot(d), q(q_and_r), r(q_and_r), active_parent(act_parent), passive_parent(NULL), tps(NULL) {}
+
+  // constructors for SCAN
+  Edge(WordID c, const EGrammarNode* d, const FSTNode* i, const FSTNode* j,
+       const Edge* act_par, const TargetPhraseSet* translations) :
+#ifdef DEBUG_CHART_PARSER
+    id(++id_count),
+#endif
+    cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(NULL), tps(translations) {}
+
+  Edge(WordID c, const EGrammarNode* d, const FSTNode* i, const FSTNode* j,
+       const Edge* act_par, const TargetPhraseSet* translations,
+       const SparseVector<double>& feats) :
+#ifdef DEBUG_CHART_PARSER
+    id(++id_count),
+#endif
+    cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(NULL), tps(translations),
+    features(new SparseVector<double>(feats)) {}
+
+  // constructors for COMPLETE
+  Edge(WordID c, const EGrammarNode* d, const FSTNode* i, const FSTNode* j,
+       const Edge* act_par, const Edge *pas_par) :
+#ifdef DEBUG_CHART_PARSER
+    id(++id_count),
+#endif
+    cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(pas_par), tps(NULL) {
+      assert(pas_par->IsPassive());
+      assert(act_par->IsActive());
+    }
+
+  Edge(WordID c, const EGrammarNode* d, const FSTNode* i, const FSTNode* j,
+       const Edge* act_par, const Edge *pas_par, const SparseVector<double>& feats) :
+#ifdef DEBUG_CHART_PARSER
+    id(++id_count),
+#endif
+    cat(c), dot(d), q(i), r(j), active_parent(act_par), passive_parent(pas_par), tps(NULL),
+    features(new SparseVector<double>(feats)) {
+      assert(pas_par->IsPassive());
+      assert(act_par->IsActive());
+    }
+
+  // constructor for COMPLETE query
+  Edge(const FSTNode* _r) :
+#ifdef DEBUG_CHART_PARSER
+    id(0),
+#endif
+    cat(0), dot(NULL), q(NULL),
+    r(_r), active_parent(NULL), passive_parent(NULL), tps(NULL) {}
+  // constructor for MERGE quere
+  Edge(const FSTNode* _q, int) :
+#ifdef DEBUG_CHART_PARSER
+    id(0),
+#endif
+    cat(0), dot(NULL), q(_q),
+    r(NULL), active_parent(NULL), passive_parent(NULL), tps(NULL) {}
+};
+#ifdef DEBUG_CHART_PARSER
+int Edge::id_count = 0;
+#endif
+
+ostream& operator<<(ostream& os, const Edge& e) {
+  string type = "PREDICT";
+  if (e.IsCreatedByScan())
+    type = "SCAN";
+  else if (e.IsCreatedByComplete())
+    type = "COMPLETE"; 
+  os << "["
+#ifdef DEBUG_CHART_PARSER
+     << '(' << e.id << ") "
+#else
+     << '(' << &e << ") "
+#endif
+     << "q=" << e.q << ", r=" << e.r 
+     << ", cat="<< TD::Convert(e.cat*-1) << ", dot=" 
+     << e.dot
+#ifdef DEBUG_CHART_PARSER
+     << e.dot->hint
+#endif
+     << (e.IsActive() ? ", Active" : ", Passive")
+     << ", " << type;
+#ifdef DEBUG_CHART_PARSER
+  if (e.active_parent) { os << ", act.parent=(" << e.active_parent->id << ')'; }
+  if (e.passive_parent) { os << ", psv.parent=(" << e.passive_parent->id << ')'; }
+#endif
+  if (e.tps) { os << ", tps=" << e.tps; }
+  return os << ']';
+}
+
+struct Traversal {
+  const Edge* const edge;      // result from the active / passive combination
+  const Edge* const active;
+  const Edge* const passive;
+  Traversal(const Edge* me, const Edge* a, const Edge* p) : edge(me), active(a), passive(p) {}
+};
+
+struct UniqueTraversalHash {
+  size_t operator()(const Traversal* t) const {
+    size_t x = 5381;
+    x = ((x << 5) + x) ^ reinterpret_cast<size_t>(t->active);
+    x = ((x << 5) + x) ^ reinterpret_cast<size_t>(t->passive);
+    x = ((x << 5) + x) ^ t->edge->IsActive();
+    return x;
+  }
+};
+
+struct UniqueTraversalEquals {
+  size_t operator()(const Traversal* a, const Traversal* b) const {
+    return (a->passive == b->passive && a->active == b->active && a->edge->IsActive() == b->edge->IsActive());
+  }
+};
+
+struct UniqueEdgeHash {
+  size_t operator()(const Edge* e) const {
+    size_t x = 5381;
+    if (e->IsActive()) {
+      x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->dot);
+      x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->q);
+      x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->r);
+      x = ((x << 5) + x) ^ static_cast<size_t>(e->cat);
+      x += 13;
+    } else {  // with passive edges, we don't care about the dot
+      x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->q);
+      x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->r);
+      x = ((x << 5) + x) ^ static_cast<size_t>(e->cat);
+    }
+    return x;
+  }
+};
+
+struct UniqueEdgeEquals {
+  bool operator()(const Edge* a, const Edge* b) const {
+    if (a->IsActive() != b->IsActive()) return false;
+    if (a->IsActive()) {
+      return (a->cat == b->cat) && (a->dot == b->dot) && (a->q == b->q) && (a->r == b->r);
+    } else {
+      return (a->cat == b->cat) && (a->q == b->q) && (a->r == b->r);
+    }
+  }
+};
+
+struct REdgeHash {
+  size_t operator()(const Edge* e) const {
+    size_t x = 5381;
+    x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->r);
+    return x;
+  }
+};
+
+struct REdgeEquals {
+  bool operator()(const Edge* a, const Edge* b) const {
+    return (a->r == b->r);
+  }
+};
+
+struct QEdgeHash {
+  size_t operator()(const Edge* e) const {
+    size_t x = 5381;
+    x = ((x << 5) + x) ^ reinterpret_cast<size_t>(e->q);
+    return x;
+  }
+};
+
+struct QEdgeEquals {
+  bool operator()(const Edge* a, const Edge* b) const {
+    return (a->q == b->q);
+  }
+};
+
+struct EdgeQueue {
+  queue<const Edge*> q;
+  EdgeQueue() {}
+  void clear() { while(!q.empty()) q.pop(); }
+  bool HasWork() const { return !q.empty(); }
+  const Edge* Next() { const Edge* res = q.front(); q.pop(); return res; }
+  void AddEdge(const Edge* s) { q.push(s); }
+};
+
+class EarleyComposerImpl {
+ public:
+  EarleyComposerImpl(WordID start_cat, const FSTNode& q_0) : start_cat_(start_cat), q_0_(&q_0) {}
+
+  // returns false if the intersection is empty
+  bool Compose(const EGrammar& g, Hypergraph* forest) {
+    goal_node = NULL;
+    EGrammar::const_iterator sit = g.find(start_cat_);
+    forest->ReserveNodes(kMAX_NODES);
+    assert(sit != g.end());
+    Edge* init = new Edge(start_cat_, &sit->second, q_0_);
+    assert(IncorporateNewEdge(init));
+    while (exp_agenda.HasWork() || agenda.HasWork()) {
+      while(exp_agenda.HasWork()) {
+        const Edge* edge = exp_agenda.Next();
+        FinishEdge(edge, forest);
+      }
+      if (agenda.HasWork()) {
+        const Edge* edge = agenda.Next();
+#ifdef DEBUG_CHART_PARSER
+        cerr << "processing (" << edge->id << ')' << endl;
+#endif
+        if (edge->IsActive()) {
+          if (edge->dot->HasTerminals())
+            DoScan(edge);
+          if (edge->dot->HasNonTerminals()) {
+            DoMergeWithPassives(edge);
+            DoPredict(edge, g);
+          }
+        } else {
+          DoComplete(edge);
+        }
+      }
+    }
+    if (goal_node) {
+      forest->PruneUnreachable(goal_node->id_);
+      forest->EpsilonRemove(kEPS);
+    }
+    FreeAll();
+    return goal_node;
+  }
+
+  void FreeAll() {
+    for (int i = 0; i < free_list_.size(); ++i)
+      delete free_list_[i];
+    free_list_.clear();
+    for (int i = 0; i < traversal_free_list_.size(); ++i)
+      delete traversal_free_list_[i];
+    traversal_free_list_.clear();
+    all_traversals.clear();
+    exp_agenda.clear();
+    agenda.clear();
+    tps2node.clear();
+    edge2node.clear();
+    all_edges.clear();
+    passive_edges.clear();
+    active_edges.clear();
+  }
+
+  ~EarleyComposerImpl() {
+    FreeAll();
+  }
+
+  // returns the total number of edges created during composition
+  int EdgesCreated() const {
+    return free_list_.size();
+  }
+
+ private:
+  void DoScan(const Edge* edge) {
+    // here, we assume that the FST will potentially have many more outgoing
+    // edges than the grammar, which will be just a couple.  If you want to
+    // efficiently handle the case where both are relatively large, this code
+    // will need to change how the intersection is done.  The best general
+    // solution would probably be the Baeza-Yates double binary search.
+
+    const EGrammarNode* dot = edge->dot;
+    const FSTNode* r = edge->r;
+    const map<WordID, EGrammarNode>& terms = dot->GetTerminals();
+    for (map<WordID, EGrammarNode>::const_iterator git = terms.begin();
+         git != terms.end(); ++git) {
+      const FSTNode* next_r = r->Extend(git->first);
+      if (!next_r) continue;
+      const EGrammarNode* next_dot = &git->second;
+      const bool grammar_continues = next_dot->GrammarContinues();
+      const bool rule_completes    = next_dot->RuleCompletes();
+      assert(grammar_continues || rule_completes);
+      const SparseVector<double>& input_features = next_dot->GetCFGProductionFeatures();
+      // create up to 4 new edges!
+      if (next_r->HasOutgoingNonEpsilonEdges()) {     // are there further symbols in the FST?
+        const TargetPhraseSet* translations = NULL;
+        if (rule_completes)
+          IncorporateNewEdge(new Edge(edge->cat, next_dot, edge->q, next_r, edge, translations, input_features));
+        if (grammar_continues)
+          IncorporateNewEdge(new Edge(edge->cat, next_dot, edge->q, next_r, edge, translations));
+      }
+      if (next_r->HasData()) {   // indicates a loop back to q_0 in the FST
+        const TargetPhraseSet* translations = next_r->GetTranslations();
+        if (rule_completes)
+          IncorporateNewEdge(new Edge(edge->cat, next_dot, edge->q, q_0_, edge, translations, input_features));
+        if (grammar_continues)
+          IncorporateNewEdge(new Edge(edge->cat, next_dot, edge->q, q_0_, edge, translations));
+      }
+    }
+  }
+
+  void DoPredict(const Edge* edge, const EGrammar& g) {
+    const EGrammarNode* dot = edge->dot;
+    const map<WordID, EGrammarNode>& non_terms = dot->GetNonTerminals();
+    for (map<WordID, EGrammarNode>::const_iterator git = non_terms.begin();
+         git != non_terms.end(); ++git) {
+      const WordID nt_to_predict = git->first;
+      //cerr << edge->id << " -- " << TD::Convert(nt_to_predict*-1) << endl;
+      EGrammar::const_iterator egi = g.find(nt_to_predict);
+      if (egi == g.end()) {
+        cerr << "[ERROR] Can't find any grammar rules with a LHS of type "
+             << TD::Convert(-1*nt_to_predict) << '!' << endl;
+        continue;
+      }
+      assert(edge->IsActive());
+      const EGrammarNode* new_dot = &egi->second;
+      Edge* new_edge = new Edge(nt_to_predict, new_dot, edge->r, edge);
+      IncorporateNewEdge(new_edge);
+    }
+  }
+
+  void DoComplete(const Edge* passive) {
+#ifdef DEBUG_CHART_PARSER
+    cerr << "  complete: " << *passive << endl;
+#endif
+    const WordID completed_nt = passive->cat;
+    const FSTNode* q = passive->q;
+    const FSTNode* next_r = passive->r;
+    const Edge query(q);
+    const pair<unordered_multiset<const Edge*, REdgeHash, REdgeEquals>::iterator,
+         unordered_multiset<const Edge*, REdgeHash, REdgeEquals>::iterator > p =
+      active_edges.equal_range(&query);
+    for (unordered_multiset<const Edge*, REdgeHash, REdgeEquals>::iterator it = p.first;
+         it != p.second; ++it) {
+      const Edge* active = *it;
+#ifdef DEBUG_CHART_PARSER
+      cerr << "    pos: " << *active << endl;
+#endif
+      const EGrammarNode* next_dot = active->dot->Extend(completed_nt);
+      if (!next_dot) continue;
+      const SparseVector<double>& input_features = next_dot->GetCFGProductionFeatures();
+      // add up to 2 rules
+      if (next_dot->RuleCompletes())
+        IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive, input_features));
+      if (next_dot->GrammarContinues())
+        IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive));
+    }
+  }
+
+  void DoMergeWithPassives(const Edge* active) {
+    // edge is active, has non-terminals, we need to find the passives that can extend it
+    assert(active->IsActive());
+    assert(active->dot->HasNonTerminals());
+#ifdef DEBUG_CHART_PARSER
+    cerr << "  merge active with passives: ACT=" << *active << endl;
+#endif
+    const Edge query(active->r, 1);
+    const pair<unordered_multiset<const Edge*, QEdgeHash, QEdgeEquals>::iterator,
+         unordered_multiset<const Edge*, QEdgeHash, QEdgeEquals>::iterator > p =
+      passive_edges.equal_range(&query);
+    for (unordered_multiset<const Edge*, QEdgeHash, QEdgeEquals>::iterator it = p.first;
+         it != p.second; ++it) {
+      const Edge* passive = *it;
+      const EGrammarNode* next_dot = active->dot->Extend(passive->cat);
+      if (!next_dot) continue;
+      const FSTNode* next_r = passive->r;
+      const SparseVector<double>& input_features = next_dot->GetCFGProductionFeatures();
+      if (next_dot->RuleCompletes())
+        IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive, input_features));
+      if (next_dot->GrammarContinues())
+        IncorporateNewEdge(new Edge(active->cat, next_dot, active->q, next_r, active, passive));
+    }
+  }
+
+  // take ownership of edge memory, add to various indexes, etc
+  // returns true if this edge is new
+  bool IncorporateNewEdge(Edge* edge) {
+    free_list_.push_back(edge);
+    if (edge->passive_parent && edge->active_parent) {
+      Traversal* t = new Traversal(edge, edge->active_parent, edge->passive_parent);
+      traversal_free_list_.push_back(t);
+      if (all_traversals.find(t) != all_traversals.end()) {
+        return false;
+      } else {
+        all_traversals.insert(t);
+      }
+    }
+    exp_agenda.AddEdge(edge);
+    return true;
+  }
+
+  bool FinishEdge(const Edge* edge, Hypergraph* hg) {
+    bool is_new = false;
+    if (all_edges.find(edge) == all_edges.end()) {
+#ifdef DEBUG_CHART_PARSER
+      cerr << *edge << " is NEW\n";
+#endif
+      all_edges.insert(edge);
+      is_new = true;
+      if (edge->IsPassive()) passive_edges.insert(edge);
+      if (edge->IsActive()) active_edges.insert(edge);
+      agenda.AddEdge(edge);
+    } else {
+#ifdef DEBUG_CHART_PARSER
+      cerr << *edge << " is NOT NEW.\n";
+#endif
+    }
+    AddEdgeToTranslationForest(edge, hg);
+    return is_new;
+  }
+
+  // build the translation forest
+  void AddEdgeToTranslationForest(const Edge* edge, Hypergraph* hg) {
+    assert(hg->nodes_.size() < kMAX_NODES);
+    Hypergraph::Node* tps = NULL;
+    // first add any target language rules
+    if (edge->tps) {
+      Hypergraph::Node*& node = tps2node[(size_t)edge->tps];
+      if (!node) {
+        // cerr << "Creating phrases for " << edge->tps << endl;
+        const vector<TRulePtr>& rules = edge->tps->GetRules();
+        node = hg->AddNode(kPHRASE);
+        for (int i = 0; i < rules.size(); ++i) {
+          Hypergraph::Edge* hg_edge = hg->AddEdge(rules[i], Hypergraph::TailNodeVector());
+          hg_edge->feature_values_ += rules[i]->GetFeatureValues();
+          hg->ConnectEdgeToHeadNode(hg_edge, node);
+        }
+      }
+      tps = node;
+    }
+    Hypergraph::Node*& head_node = edge2node[edge];
+    if (!head_node)
+      head_node = hg->AddNode(kPHRASE);
+    if (edge->cat == start_cat_ && edge->q == q_0_ && edge->r == q_0_ && edge->IsPassive()) {
+      assert(goal_node == NULL || goal_node == head_node);
+      goal_node = head_node;
+    }
+    Hypergraph::TailNodeVector tail;
+    SparseVector<double> extra;
+    if (edge->IsCreatedByPredict()) {
+      // extra.set_value(FD::Convert("predict"), 1);
+    } else if (edge->IsCreatedByScan()) {
+      tail.push_back(edge2node[edge->active_parent]->id_);
+      if (tps) {
+        tail.push_back(tps->id_);
+      }
+      //extra.set_value(FD::Convert("scan"), 1);
+    } else if (edge->IsCreatedByComplete()) {
+      tail.push_back(edge2node[edge->active_parent]->id_);
+      tail.push_back(edge2node[edge->passive_parent]->id_);
+      //extra.set_value(FD::Convert("complete"), 1);
+    } else {
+      assert(!"unexpected edge type!");
+    }
+    //cerr << head_node->id_ << "<--" << *edge << endl;
+
+#ifdef DEBUG_CHART_PARSER
+      for (int i = 0; i < tail.size(); ++i)
+        if (tail[i] == head_node->id_) {
+          cerr << "ERROR: " << *edge << "\n   i=" << i << endl;
+          if (i == 1) { cerr << "\tP: " << *edge->passive_parent << endl; }
+          if (i == 0) { cerr << "\tA: " << *edge->active_parent << endl; }
+          assert(!"self-loop found!");
+        }
+#endif
+    Hypergraph::Edge* hg_edge = NULL; 
+    if (tail.size() == 0) {
+      hg_edge = hg->AddEdge(kEPSRule, tail);
+    } else if (tail.size() == 1) {
+      hg_edge = hg->AddEdge(kX1, tail);
+    } else if (tail.size() == 2) {
+      hg_edge = hg->AddEdge(kX1X2, tail);
+    }
+    if (edge->features)
+      hg_edge->feature_values_ += *edge->features;
+    hg_edge->feature_values_ += extra;
+    hg->ConnectEdgeToHeadNode(hg_edge, head_node);
+  }
+
+  Hypergraph::Node* goal_node;
+  EdgeQueue exp_agenda;
+  EdgeQueue agenda;
+  unordered_map<size_t, Hypergraph::Node*> tps2node;
+  unordered_map<const Edge*, Hypergraph::Node*, UniqueEdgeHash, UniqueEdgeEquals> edge2node;
+  unordered_set<const Traversal*, UniqueTraversalHash, UniqueTraversalEquals> all_traversals;
+  unordered_set<const Edge*, UniqueEdgeHash, UniqueEdgeEquals> all_edges;
+  unordered_multiset<const Edge*, QEdgeHash, QEdgeEquals> passive_edges;
+  unordered_multiset<const Edge*, REdgeHash, REdgeEquals> active_edges;
+  vector<Edge*> free_list_;
+  vector<Traversal*> traversal_free_list_;
+  const WordID start_cat_;
+  const FSTNode* const q_0_;
+};
+
+#ifdef DEBUG_CHART_PARSER
+static string TrimRule(const string& r) {
+  size_t start = r.find(" |||") + 5;
+  size_t end = r.rfind(" |||");
+  return r.substr(start, end - start);
+}
+#endif
+
+void AddGrammarRule(const string& r, EGrammar* g) {
+  const size_t pos = r.find(" ||| ");
+  if (pos == string::npos || r[0] != '[') {
+    cerr << "Bad rule: " << r << endl;
+    return;
+  }
+  const size_t rpos = r.rfind(" ||| ");
+  string feats;
+  string rs = r;
+  if (rpos != pos) {
+    feats = r.substr(rpos + 5);
+    rs = r.substr(0, rpos);
+  }
+  string rhs = rs.substr(pos + 5);
+  string trule = rs + " ||| " + rhs + " ||| " + feats;
+  TRule tr(trule);
+#ifdef DEBUG_CHART_PARSER
+  string hint_last_rule;
+#endif
+  EGrammarNode* cur = &(*g)[tr.GetLHS()];
+  cur->is_root = true;
+  for (int i = 0; i < tr.FLength(); ++i) {
+    WordID sym = tr.f()[i];
+#ifdef DEBUG_CHART_PARSER
+    hint_last_rule = TD::Convert(sym < 0 ? -sym : sym);
+    cur->hint += " <@@> (*" + hint_last_rule + ") " + TrimRule(tr.AsString());
+#endif
+    if (sym < 0)
+      cur = &cur->ntptr[sym];
+    else
+      cur = &cur->tptr[sym];
+  }
+#ifdef DEBUG_CHART_PARSER
+  cur->hint += " <@@> (" + hint_last_rule + "*) " + TrimRule(tr.AsString());
+#endif
+  cur->is_some_rule_complete = true;
+  cur->input_features = tr.GetFeatureValues();
+}
+
+EarleyComposer::~EarleyComposer() {
+  delete pimpl_;
+}
+
+EarleyComposer::EarleyComposer(const FSTNode* fst) {
+  InitializeConstants();
+  pimpl_ = new EarleyComposerImpl(kUNIQUE_START, *fst);
+}
+
+bool EarleyComposer::Compose(const Hypergraph& src_forest, Hypergraph* trg_forest) {
+  // first, convert the src forest into an EGrammar
+  EGrammar g;
+  const int nedges = src_forest.edges_.size();
+  const int nnodes = src_forest.nodes_.size();
+  vector<int> cats(nnodes);
+  bool assign_cats = false;
+  for (int i = 0; i < nnodes; ++i)
+    if (assign_cats) {
+      cats[i] = TD::Convert("CAT_" + boost::lexical_cast<string>(i)) * -1;
+    } else {
+      cats[i] = src_forest.nodes_[i].cat_;
+    }
+  // construct the grammar
+  for (int i = 0; i < nedges; ++i) {
+    const Hypergraph::Edge& edge = src_forest.edges_[i];
+    const vector<WordID>& src = edge.rule_->f();
+    EGrammarNode* cur = &g[cats[edge.head_node_]];
+    cur->is_root = true;
+    int ntc = 0;
+    for (int j = 0; j < src.size(); ++j) {
+      WordID sym = src[j];
+      if (sym <= 0) {
+        sym = cats[edge.tail_nodes_[ntc]];
+        ++ntc;
+        cur = &cur->ntptr[sym];
+      } else {
+        cur = &cur->tptr[sym];
+      }
+    }
+    cur->is_some_rule_complete = true;
+    cur->input_features = edge.feature_values_;
+  }
+  EGrammarNode& goal_rule = g[kUNIQUE_START];
+  assert((goal_rule.ntptr.size() == 1 && goal_rule.tptr.size() == 0) ||
+         (goal_rule.ntptr.size() == 0 && goal_rule.tptr.size() == 1));
+
+  return pimpl_->Compose(g, trg_forest);
+}
+
+bool EarleyComposer::Compose(istream* in, Hypergraph* trg_forest) {
+  EGrammar g;
+  while(*in) {
+    string line;
+    getline(*in, line);
+    if (line.empty()) continue;
+    AddGrammarRule(line, &g);
+  }
+
+  return pimpl_->Compose(g, trg_forest);
+}
diff --git a/decoder/earley_composer.h b/decoder/earley_composer.h
new file mode 100644
index 00000000..9f786bf6
--- /dev/null
+++ b/decoder/earley_composer.h
@@ -0,0 +1,29 @@
+#ifndef _EARLEY_COMPOSER_H_
+#define _EARLEY_COMPOSER_H_
+
+#include <iostream>
+
+class EarleyComposerImpl;
+class FSTNode;
+class Hypergraph;
+
+class EarleyComposer {
+ public:
+  ~EarleyComposer();
+  EarleyComposer(const FSTNode* phrasetable_root);
+  bool Compose(const Hypergraph& src_forest, Hypergraph* trg_forest);
+
+  // reads the grammar from a file. There must be a single top-level
+  // S -> X rule.  Anything else is possible. Format is:
+  // [S] ||| [SS,1]
+  // [SS] ||| [NP,1] [VP,2] ||| Feature1=0.2 Feature2=-2.3
+  // [SS] ||| [VP,1] [NP,2] ||| Feature1=0.8
+  // [NP] ||| [DET,1] [N,2] ||| Feature3=2
+  // ...
+  bool Compose(std::istream* grammar_file, Hypergraph* trg_forest);
+
+ private:
+  EarleyComposerImpl* pimpl_;
+};
+
+#endif
diff --git a/decoder/exp_semiring.h b/decoder/exp_semiring.h
new file mode 100644
index 00000000..f91beee4
--- /dev/null
+++ b/decoder/exp_semiring.h
@@ -0,0 +1,71 @@
+#ifndef _EXP_SEMIRING_H_
+#define _EXP_SEMIRING_H_
+
+#include <iostream>
+
+// this file implements the first-order expectation semiring described
+// in Li & Eisner (EMNLP 2009)
+
+// requirements:
+//   RType * RType ==> RType
+//   PType * PType ==> PType
+//   RType * PType ==> RType
+// good examples:
+//   PType scalar, RType vector
+// BAD examples:
+//   PType vector, RType scalar
+template <typename PType, typename RType>
+struct PRPair {
+  PRPair() : p(), r() {}
+  // Inside algorithm requires that T(0) and T(1)
+  // return the 0 and 1 values of the semiring
+  explicit PRPair(double x) : p(x), r() {}
+  PRPair(const PType& p, const RType& r) : p(p), r(r) {}
+  PRPair& operator+=(const PRPair& o) {
+    p += o.p;
+    r += o.r;
+    return *this;
+  }
+  PRPair& operator*=(const PRPair& o) {
+    r = (o.r * p) + (o.p * r);
+    p *= o.p;
+    return *this;
+  }
+  PType p;
+  RType r;
+};
+
+template <typename P, typename R>
+std::ostream& operator<<(std::ostream& o, const PRPair<P,R>& x) {
+  return o << '<' << x.p << ", " << x.r << '>';
+}
+
+template <typename P, typename R>
+const PRPair<P,R> operator+(const PRPair<P,R>& a, const PRPair<P,R>& b) {
+  PRPair<P,R> result = a;
+  result += b;
+  return result;
+}
+
+template <typename P, typename R>
+const PRPair<P,R> operator*(const PRPair<P,R>& a, const PRPair<P,R>& b) {
+  PRPair<P,R> result = a;
+  result *= b;
+  return result;
+}
+
+template <typename P, typename PWeightFunction, typename R, typename RWeightFunction>
+struct PRWeightFunction {
+  explicit PRWeightFunction(const PWeightFunction& pwf = PWeightFunction(),
+                            const RWeightFunction& rwf = RWeightFunction()) :
+    pweight(pwf), rweight(rwf) {}
+  PRPair<P,R> operator()(const Hypergraph::Edge& e) const {
+    const P p = pweight(e);
+    const R r = rweight(e);
+    return PRPair<P,R>(p, r * p);
+  }
+  const PWeightFunction pweight;
+  const RWeightFunction rweight;
+};
+
+#endif
diff --git a/decoder/fdict.cc b/decoder/fdict.cc
new file mode 100644
index 00000000..7e1b0e1f
--- /dev/null
+++ b/decoder/fdict.cc
@@ -0,0 +1,129 @@
+#include "fdict.h"
+
+#include <string>
+
+using namespace std;
+
+Dict FD::dict_;
+bool FD::frozen_ = false;
+
+static int HexPairValue(const char * code) {
+  int value = 0;
+  const char * pch = code;
+  for (;;) {
+    int digit = *pch++;
+    if (digit >= '0' && digit <= '9') {
+      value += digit - '0';
+    }
+    else if (digit >= 'A' && digit <= 'F') {
+      value += digit - 'A' + 10;
+    }
+    else if (digit >= 'a' && digit <= 'f') {
+      value += digit - 'a' + 10;
+    }
+    else {
+      return -1;
+    }
+    if (pch == code + 2)
+      return value;
+    value <<= 4;
+  }
+}
+
+int UrlDecode(const char *source, char *dest)
+{
+  char * start = dest;
+
+  while (*source) {
+    switch (*source) {
+    case '+':
+      *(dest++) = ' ';
+      break;
+    case '%':
+      if (source[1] && source[2]) {
+        int value = HexPairValue(source + 1);
+        if (value >= 0) {
+          *(dest++) = value;
+          source += 2;
+        }
+        else {
+          *dest++ = '?';
+        }
+      }
+      else {
+        *dest++ = '?';
+      }
+      break;
+    default:
+      *dest++ = *source;
+    }
+    source++;
+  }
+  
+  *dest = 0;
+  return dest - start;
+}  
+
+int UrlEncode(const char *source, char *dest, unsigned max) {
+  static const char *digits = "0123456789ABCDEF";
+  unsigned char ch;
+  unsigned len = 0;
+  char *start = dest;
+
+  while (len < max - 4 && *source)
+  {
+    ch = (unsigned char)*source;
+    if (*source == ' ') {
+      *dest++ = '+';
+    }
+    else if (strchr("=:;,_| %", ch)) {
+      *dest++ = '%';
+      *dest++ = digits[(ch >> 4) & 0x0F];
+      *dest++ = digits[       ch & 0x0F];
+    }
+    else {
+      *dest++ = *source;
+    }  
+    source++;
+  }
+  *dest = 0;
+  return start - dest;
+}
+
+std::string UrlDecodeString(const std::string & encoded) {
+  const char * sz_encoded = encoded.c_str();
+  size_t needed_length = encoded.length();
+  for (const char * pch = sz_encoded; *pch; pch++) {
+    if (*pch == '%')
+      needed_length += 2;
+  }
+  needed_length += 10;
+  char stackalloc[64];
+  char * buf = needed_length > sizeof(stackalloc)/sizeof(*stackalloc) ?
+    (char *)malloc(needed_length) : stackalloc;
+  UrlDecode(encoded.c_str(), buf);
+  std::string result(buf);
+  if (buf != stackalloc) {
+    free(buf);
+  }
+  return result;
+}
+
+std::string UrlEncodeString(const std::string & decoded) {
+  const char * sz_decoded = decoded.c_str();
+  size_t needed_length = decoded.length() * 3 + 3;
+  char stackalloc[64];
+  char * buf = needed_length > sizeof(stackalloc)/sizeof(*stackalloc) ?
+    (char *)malloc(needed_length) : stackalloc;
+  UrlEncode(decoded.c_str(), buf, needed_length);
+  std::string result(buf);
+  if (buf != stackalloc) {
+    free(buf);
+  }
+  return result;
+}
+
+string FD::Escape(const string& s) {
+  return UrlEncodeString(s);
+}
+
diff --git a/decoder/fdict.h b/decoder/fdict.h
new file mode 100644
index 00000000..c4236580
--- /dev/null
+++ b/decoder/fdict.h
@@ -0,0 +1,31 @@
+#ifndef _FDICT_H_
+#define _FDICT_H_
+
+#include <string>
+#include <vector>
+#include "dict.h"
+
+struct FD {
+  // once the FD is frozen, new features not already in the
+  // dictionary will return 0
+  static void Freeze() {
+    frozen_ = true;
+  }
+  static inline int NumFeats() {
+    return dict_.max() + 1;
+  }
+  static inline WordID Convert(const std::string& s) {
+    return dict_.Convert(s, frozen_);
+  }
+  static inline const std::string& Convert(const WordID& w) {
+    return dict_.Convert(w);
+  }
+  // Escape any string to a form that can be used as the name
+  // of a weight in a weights file
+  static std::string Escape(const std::string& s);
+  static Dict dict_;
+ private:
+  static bool frozen_;
+};
+
+#endif
diff --git a/decoder/ff.cc b/decoder/ff.cc
new file mode 100644
index 00000000..61f4f0b6
--- /dev/null
+++ b/decoder/ff.cc
@@ -0,0 +1,137 @@
+#include "ff.h"
+
+#include "tdict.h"
+#include "hg.h"
+
+using namespace std;
+
+FeatureFunction::~FeatureFunction() {}
+
+
+void FeatureFunction::FinalTraversalFeatures(const void* ant_state,
+                                             SparseVector<double>* features) const {
+  (void) ant_state;
+  (void) features;
+}
+
+// Hiero and Joshua use log_10(e) as the value, so I do to
+WordPenalty::WordPenalty(const string& param) :
+    fid_(FD::Convert("WordPenalty")),
+    value_(-1.0 / log(10)) {
+  if (!param.empty()) {
+    cerr << "Warning WordPenalty ignoring parameter: " << param << endl;
+  }
+}
+
+void WordPenalty::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                        const Hypergraph::Edge& edge,
+                                        const std::vector<const void*>& ant_states,
+                                        SparseVector<double>* features,
+                                        SparseVector<double>* estimated_features,
+                                        void* state) const {
+  (void) smeta;
+  (void) ant_states;
+  (void) state;
+  (void) estimated_features;
+  features->set_value(fid_, edge.rule_->EWords() * value_);
+}
+
+SourceWordPenalty::SourceWordPenalty(const string& param) :
+    fid_(FD::Convert("SourceWordPenalty")),
+    value_(-1.0 / log(10)) {
+  if (!param.empty()) {
+    cerr << "Warning SourceWordPenalty ignoring parameter: " << param << endl;
+  }
+}
+
+void SourceWordPenalty::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                        const Hypergraph::Edge& edge,
+                                        const std::vector<const void*>& ant_states,
+                                        SparseVector<double>* features,
+                                        SparseVector<double>* estimated_features,
+                                        void* state) const {
+  (void) smeta;
+  (void) ant_states;
+  (void) state;
+  (void) estimated_features;
+  features->set_value(fid_, edge.rule_->FWords() * value_);
+}
+
+ArityPenalty::ArityPenalty(const std::string& param) :
+    value_(-1.0 / log(10)) {
+  string fname = "Arity_X";
+  for (int i = 0; i < 10; ++i) {
+    fname[6]=i + '0';
+    fids_[i] = FD::Convert(fname);
+  }
+}
+
+void ArityPenalty::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                         const Hypergraph::Edge& edge,
+                                         const std::vector<const void*>& ant_states,
+                                         SparseVector<double>* features,
+                                         SparseVector<double>* estimated_features,
+                                         void* state) const {
+  (void) smeta;
+  (void) ant_states;
+  (void) state;
+  (void) estimated_features;
+  features->set_value(fids_[edge.Arity()], value_);
+}
+
+ModelSet::ModelSet(const vector<double>& w, const vector<const FeatureFunction*>& models) :
+    models_(models),
+    weights_(w),
+    state_size_(0),
+    model_state_pos_(models.size()) {
+  for (int i = 0; i < models_.size(); ++i) {
+    model_state_pos_[i] = state_size_;
+    state_size_ += models_[i]->NumBytesContext();
+  }
+}
+
+void ModelSet::AddFeaturesToEdge(const SentenceMetadata& smeta,
+                                 const Hypergraph& hg,
+                                 const vector<string>& node_states,
+                                 Hypergraph::Edge* edge,
+                                 string* context,
+                                 prob_t* combination_cost_estimate) const {
+  context->resize(state_size_);
+  memset(&(*context)[0], 0, state_size_);
+  SparseVector<double> est_vals;  // only computed if combination_cost_estimate is non-NULL
+  if (combination_cost_estimate) *combination_cost_estimate = prob_t::One();
+  for (int i = 0; i < models_.size(); ++i) {
+    const FeatureFunction& ff = *models_[i];
+    void* cur_ff_context = NULL;
+    vector<const void*> ants(edge->tail_nodes_.size());
+    bool has_context = ff.NumBytesContext() > 0;
+    if (has_context) {
+      int spos = model_state_pos_[i];
+      cur_ff_context = &(*context)[spos];
+      for (int i = 0; i < ants.size(); ++i) {
+        ants[i] = &node_states[edge->tail_nodes_[i]][spos];
+      }
+    }
+    ff.TraversalFeatures(smeta, *edge, ants, &edge->feature_values_, &est_vals, cur_ff_context);
+  }
+  if (combination_cost_estimate)
+    combination_cost_estimate->logeq(est_vals.dot(weights_));
+  edge->edge_prob_.logeq(edge->feature_values_.dot(weights_));
+}
+
+void ModelSet::AddFinalFeatures(const std::string& state, Hypergraph::Edge* edge) const {
+  assert(1 == edge->rule_->Arity());
+
+  for (int i = 0; i < models_.size(); ++i) {
+    const FeatureFunction& ff = *models_[i];
+    const void* ant_state = NULL;
+    bool has_context = ff.NumBytesContext() > 0;
+    if (has_context) {
+      int spos = model_state_pos_[i];
+      ant_state = &state[spos];
+    }
+    ff.FinalTraversalFeatures(ant_state, &edge->feature_values_);
+  }
+  edge->edge_prob_.logeq(edge->feature_values_.dot(weights_));
+}
+
diff --git a/decoder/ff.h b/decoder/ff.h
new file mode 100644
index 00000000..630b3208
--- /dev/null
+++ b/decoder/ff.h
@@ -0,0 +1,152 @@
+#ifndef _FF_H_
+#define _FF_H_
+
+#include <vector>
+
+#include "fdict.h"
+#include "hg.h"
+
+class SentenceMetadata;
+class FeatureFunction;  // see definition below
+
+// if you want to develop a new feature, inherit from this class and
+// override TraversalFeaturesImpl(...).  If it's a feature that returns /
+// depends on context, you may also need to implement
+// FinalTraversalFeatures(...)
+class FeatureFunction {
+ public:
+  FeatureFunction() : state_size_() {}
+  explicit FeatureFunction(int state_size) : state_size_(state_size) {}
+  virtual ~FeatureFunction();
+
+  // returns the number of bytes of context that this feature function will
+  // (maximally) use.  By default, 0 ("stateless" models in Hiero/Joshua).
+  // NOTE: this value is fixed for the instance of your class, you cannot
+  // use different amounts of memory for different nodes in the forest.
+  inline int NumBytesContext() const { return state_size_; }
+
+  // Compute the feature values and (if this applies) the estimates of the
+  // feature values when this edge is used incorporated into a larger context
+  inline void TraversalFeatures(const SentenceMetadata& smeta,
+                                const Hypergraph::Edge& edge,
+                                const std::vector<const void*>& ant_contexts,
+                                SparseVector<double>* features,
+                                SparseVector<double>* estimated_features,
+                                void* out_state) const {
+    TraversalFeaturesImpl(smeta, edge, ant_contexts,
+                          features, estimated_features, out_state);
+    // TODO it's easy for careless feature function developers to overwrite
+    // the end of their state and clobber someone else's memory.  These bugs
+    // will be horrendously painful to track down.  There should be some
+    // optional strict mode that's enforced here that adds some kind of
+    // barrier between the blocks reserved for the residual contexts
+  }
+
+  // if there's some state left when you transition to the goal state, score
+  // it here.  For example, the language model computes the cost of adding
+  // <s> and </s>.
+  virtual void FinalTraversalFeatures(const void* residual_state,
+                                      SparseVector<double>* final_features) const;
+
+ protected:
+  // context is a pointer to a buffer of size NumBytesContext() that the
+  // feature function can write its state to.  It's up to the feature function
+  // to determine how much space it needs and to determine how to encode its
+  // residual contextual information since it is OPAQUE to all clients outside
+  // of the particular FeatureFunction class.  There is one exception:
+  // equality of the contents (i.e., memcmp) is required to determine whether
+  // two states can be combined.
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const = 0;
+
+  // !!! ONLY call this from subclass *CONSTRUCTORS* !!!
+  void SetStateSize(size_t state_size) {
+    state_size_ = state_size;
+  }
+
+ private:
+  int state_size_;
+};
+
+// word penalty feature, for each word on the E side of a rule,
+// add value_
+class WordPenalty : public FeatureFunction {
+ public:
+  WordPenalty(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const;
+ private:
+  const int fid_;
+  const double value_;
+};
+
+class SourceWordPenalty : public FeatureFunction {
+ public:
+  SourceWordPenalty(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const;
+ private:
+  const int fid_;
+  const double value_;
+};
+
+class ArityPenalty : public FeatureFunction {
+ public:
+  ArityPenalty(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const;
+ private:
+  int fids_[10];
+  const double value_;
+};
+
+// this class is a set of FeatureFunctions that can be used to score, rescore,
+// etc. a (translation?) forest
+class ModelSet {
+ public:
+  ModelSet() : state_size_(0) {}
+
+  ModelSet(const std::vector<double>& weights,
+           const std::vector<const FeatureFunction*>& models);
+
+  // sets edge->feature_values_ and edge->edge_prob_
+  // NOTE: edge must not necessarily be in hg.edges_ but its TAIL nodes
+  // must be.
+  void AddFeaturesToEdge(const SentenceMetadata& smeta,
+                         const Hypergraph& hg,
+                         const std::vector<std::string>& node_states,
+                         Hypergraph::Edge* edge,
+                         std::string* residual_context,
+                         prob_t* combination_cost_estimate = NULL) const;
+
+  void AddFinalFeatures(const std::string& residual_context,
+                        Hypergraph::Edge* edge) const;
+
+  bool empty() const { return models_.empty(); }
+ private:
+  std::vector<const FeatureFunction*> models_;
+  std::vector<double> weights_; 
+  int state_size_;
+  std::vector<int> model_state_pos_;
+};
+
+#endif
diff --git a/decoder/ff_csplit.cc b/decoder/ff_csplit.cc
new file mode 100644
index 00000000..658603e4
--- /dev/null
+++ b/decoder/ff_csplit.cc
@@ -0,0 +1,225 @@
+#include "ff_csplit.h"
+
+#include <set>
+#include <cstring>
+
+#include "Vocab.h"
+#include "Ngram.h"
+
+#include "sentence_metadata.h"
+#include "lattice.h"
+#include "tdict.h"
+#include "freqdict.h"
+#include "filelib.h"
+#include "stringlib.h"
+#include "tdict.h"
+
+using namespace std;
+
+struct BasicCSplitFeaturesImpl {
+  BasicCSplitFeaturesImpl(const string& param) :
+      word_count_(FD::Convert("WordCount")),
+      letters_sq_(FD::Convert("LettersSq")),
+      letters_sqrt_(FD::Convert("LettersSqrt")),
+      in_dict_(FD::Convert("InDict")),
+      short_(FD::Convert("Short")),
+      long_(FD::Convert("Long")),
+      oov_(FD::Convert("OOV")),
+      short_range_(FD::Convert("ShortRange")),
+      high_freq_(FD::Convert("HighFreq")),
+      med_freq_(FD::Convert("MedFreq")),
+      freq_(FD::Convert("Freq")),
+      fl1_(FD::Convert("FreqLen1")),
+      fl2_(FD::Convert("FreqLen2")),
+      bad_(FD::Convert("Bad")) {
+    vector<string> argv;
+    int argc = SplitOnWhitespace(param, &argv);
+    if (argc != 1 && argc != 2) {
+      cerr << "Expected: freqdict.txt [badwords.txt]\n";
+      abort();
+    }
+    freq_dict_.Load(argv[0]);
+    if (argc == 2) {
+      ReadFile rf(argv[1]);
+      istream& in = *rf.stream();
+      while(in) {
+        string badword;
+        in >> badword;
+        if (badword.empty()) continue;
+        bad_words_.insert(TD::Convert(badword));
+      }
+    }
+  }
+
+  void TraversalFeaturesImpl(const Hypergraph::Edge& edge,
+                             SparseVector<double>* features) const;
+
+  const int word_count_;
+  const int letters_sq_;
+  const int letters_sqrt_;
+  const int in_dict_;
+  const int short_;
+  const int long_;
+  const int oov_;
+  const int short_range_;
+  const int high_freq_;
+  const int med_freq_;
+  const int freq_;
+  const int fl1_;
+  const int fl2_;
+  const int bad_;
+  FreqDict freq_dict_;
+  set<WordID> bad_words_;
+};
+
+BasicCSplitFeatures::BasicCSplitFeatures(const string& param) :
+  pimpl_(new BasicCSplitFeaturesImpl(param)) {}
+
+void BasicCSplitFeaturesImpl::TraversalFeaturesImpl(
+                                     const Hypergraph::Edge& edge,
+                                     SparseVector<double>* features) const {
+  features->set_value(word_count_, 1.0);
+  features->set_value(letters_sq_, (edge.j_ - edge.i_) * (edge.j_ - edge.i_));
+  features->set_value(letters_sqrt_, sqrt(edge.j_ - edge.i_));
+  const WordID word = edge.rule_->e_[1];
+  const char* sword = TD::Convert(word);
+  const int len = strlen(sword);
+  int cur = 0;
+  int chars = 0;
+  while(cur < len) {
+    cur += UTF8Len(sword[cur]);
+    ++chars;
+  }
+
+  // these are corrections that attempt to make chars
+  // more like a phoneme count than a letter count, they
+  // are only really meaningful for german and should
+  // probably be gotten rid of
+  bool has_sch = strstr(sword, "sch");
+  bool has_ch = (!has_sch && strstr(sword, "ch"));
+  bool has_ie = strstr(sword, "ie");
+  bool has_zw = strstr(sword, "zw");
+  if (has_sch) chars -= 2;
+  if (has_ch) --chars;
+  if (has_ie) --chars;
+  if (has_zw) --chars;
+
+  float freq = freq_dict_.LookUp(word);
+  if (freq) {
+    features->set_value(freq_, freq);
+    features->set_value(in_dict_, 1.0);
+  } else {
+    features->set_value(oov_, 1.0);
+    freq = 99.0f;
+  }
+  if (bad_words_.count(word) != 0)
+    features->set_value(bad_, 1.0);
+  if (chars < 5)
+    features->set_value(short_, 1.0);
+  if (chars > 10)
+    features->set_value(long_, 1.0);
+  if (freq < 7.0f)
+    features->set_value(high_freq_, 1.0);
+  if (freq > 8.0f && freq < 10.f)
+    features->set_value(med_freq_, 1.0);
+  if (freq < 10.0f && chars < 5)
+    features->set_value(short_range_, 1.0);
+
+  // i don't understand these features, but they really help!
+  features->set_value(fl1_, sqrt(chars * freq));
+  features->set_value(fl2_, freq / chars);
+}
+
+void BasicCSplitFeatures::TraversalFeaturesImpl(
+                                     const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const {
+  (void) smeta;
+  (void) ant_contexts;
+  (void) out_context;
+  (void) estimated_features;
+  if (edge.Arity() == 0) return;
+  if (edge.rule_->EWords() != 1) return;
+  pimpl_->TraversalFeaturesImpl(edge, features);
+}
+
+struct ReverseCharLMCSplitFeatureImpl {
+  ReverseCharLMCSplitFeatureImpl(const string& param) :
+      order_(5),
+      vocab_(*TD::dict_),
+      ngram_(vocab_, order_) {
+    kBOS = vocab_.getIndex("<s>");
+    kEOS = vocab_.getIndex("</s>");
+    File file(param.c_str(), "r", 0);
+    assert(file);
+    cerr << "Reading " << order_ << "-gram LM from " << param << endl;
+    ngram_.read(file);
+  }
+
+  double LeftPhonotacticProb(const Lattice& inword, const int start) {
+    const int end = inword.size();
+    for (int i = 0; i < order_; ++i)
+      sc[i] = kBOS;
+    int sp = min(end - start, order_ - 1);
+    // cerr << "[" << start << "," << sp << "]\n";
+    int ci = (order_ - sp - 1);
+    int wi = start;
+    while (sp > 0) {
+      sc[ci] = inword[wi][0].label;
+      // cerr << " CHAR: " << TD::Convert(sc[ci]) << "  ci=" << ci << endl;
+      ++wi;
+      ++ci;
+      --sp;
+    }
+    // cerr << "  END ci=" << ci << endl;
+    sc[ci] = Vocab_None;
+    const double startprob = ngram_.wordProb(kEOS, sc);
+    // cerr << "  PROB=" << startprob << endl;
+    return startprob;
+  }
+ private:
+  const int order_;
+  Vocab& vocab_;
+  VocabIndex kBOS;
+  VocabIndex kEOS;
+  Ngram ngram_;
+  VocabIndex sc[80];
+};
+
+ReverseCharLMCSplitFeature::ReverseCharLMCSplitFeature(const string& param) :
+  pimpl_(new ReverseCharLMCSplitFeatureImpl(param)),
+  fid_(FD::Convert("RevCharLM")) {}
+
+void ReverseCharLMCSplitFeature::TraversalFeaturesImpl(
+                                     const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const {
+  (void) ant_contexts;
+  (void) estimated_features;
+  (void) out_context;
+
+  if (edge.Arity() != 1) return;
+  if (edge.rule_->EWords() != 1) return;
+  const double lpp = pimpl_->LeftPhonotacticProb(smeta.GetSourceLattice(), edge.i_);
+  features->set_value(fid_, lpp);
+#if 0
+  WordID neighbor_word = 0;
+  const WordID word = edge.rule_->e_[1];
+  if (chars > 4 && (sword[0] == 's' || sword[0] == 'n')) {
+    neighbor_word = TD::Convert(string(&sword[1]));
+  }
+  if (neighbor_word) {
+    float nfreq = freq_dict_.LookUp(neighbor_word);
+    cerr << "COMPARE: " << TD::Convert(word) << " & " << TD::Convert(neighbor_word) << endl;
+    if (!nfreq) nfreq = 99.0f;
+    features->set_value(fdoes_deletion_help_, (freq - nfreq));
+  }
+#endif
+}
+
diff --git a/decoder/ff_csplit.h b/decoder/ff_csplit.h
new file mode 100644
index 00000000..c1cfb64b
--- /dev/null
+++ b/decoder/ff_csplit.h
@@ -0,0 +1,39 @@
+#ifndef _FF_CSPLIT_H_
+#define _FF_CSPLIT_H_
+
+#include <boost/shared_ptr.hpp>
+
+#include "ff.h"
+
+class BasicCSplitFeaturesImpl;
+class BasicCSplitFeatures : public FeatureFunction {
+ public:
+  BasicCSplitFeatures(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  boost::shared_ptr<BasicCSplitFeaturesImpl> pimpl_;
+};
+
+class ReverseCharLMCSplitFeatureImpl;
+class ReverseCharLMCSplitFeature : public FeatureFunction {
+ public:
+  ReverseCharLMCSplitFeature(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  boost::shared_ptr<ReverseCharLMCSplitFeatureImpl> pimpl_;
+  const int fid_;
+};
+
+#endif
diff --git a/decoder/ff_factory.cc b/decoder/ff_factory.cc
new file mode 100644
index 00000000..1854e0bb
--- /dev/null
+++ b/decoder/ff_factory.cc
@@ -0,0 +1,35 @@
+#include "ff_factory.h"
+
+#include "ff.h"
+
+using boost::shared_ptr;
+using namespace std;
+
+FFFactoryBase::~FFFactoryBase() {}
+
+void FFRegistry::DisplayList() const {
+  for (map<string, shared_ptr<FFFactoryBase> >::const_iterator it = reg_.begin();
+       it != reg_.end(); ++it) {
+    cerr << "  " << it->first << endl;
+  }
+}
+
+shared_ptr<FeatureFunction> FFRegistry::Create(const string& ffname, const string& param) const {
+  map<string, shared_ptr<FFFactoryBase> >::const_iterator it = reg_.find(ffname);
+  shared_ptr<FeatureFunction> res;
+  if (it == reg_.end()) {
+    cerr << "I don't know how to create feature " << ffname << endl;
+  } else {
+    res = it->second->Create(param);
+  }
+  return res;
+}
+
+void FFRegistry::Register(const string& ffname, FFFactoryBase* factory) {
+  if (reg_.find(ffname) != reg_.end()) {
+    cerr << "Duplicate registration of FeatureFunction with name " << ffname << "!\n";
+    abort();
+  }
+  reg_[ffname].reset(factory);
+}
+
diff --git a/decoder/ff_factory.h b/decoder/ff_factory.h
new file mode 100644
index 00000000..bc586567
--- /dev/null
+++ b/decoder/ff_factory.h
@@ -0,0 +1,39 @@
+#ifndef _FF_FACTORY_H_
+#define _FF_FACTORY_H_
+
+#include <iostream>
+#include <string>
+#include <map>
+
+#include <boost/shared_ptr.hpp>
+
+class FeatureFunction;
+class FFRegistry;
+class FFFactoryBase;
+extern boost::shared_ptr<FFRegistry> global_ff_registry;
+
+class FFRegistry {
+  friend int main(int argc, char** argv);
+  friend class FFFactoryBase;
+ public:
+  boost::shared_ptr<FeatureFunction> Create(const std::string& ffname, const std::string& param) const;
+  void DisplayList() const;
+  void Register(const std::string& ffname, FFFactoryBase* factory);
+ private:
+  FFRegistry() {}
+  std::map<std::string, boost::shared_ptr<FFFactoryBase> > reg_;
+};
+
+struct FFFactoryBase {
+  virtual ~FFFactoryBase();
+  virtual boost::shared_ptr<FeatureFunction> Create(const std::string& param) const = 0;
+};
+
+template<class FF>
+class FFFactory : public FFFactoryBase {
+  boost::shared_ptr<FeatureFunction> Create(const std::string& param) const {
+    return boost::shared_ptr<FeatureFunction>(new FF(param));
+  }
+};
+
+#endif
diff --git a/decoder/ff_lm.cc b/decoder/ff_lm.cc
new file mode 100644
index 00000000..a1dc8b81
--- /dev/null
+++ b/decoder/ff_lm.cc
@@ -0,0 +1,454 @@
+#include "ff_lm.h"
+
+#include <sstream>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <netdb.h>
+
+#include <boost/shared_ptr.hpp>
+
+#include "tdict.h"
+#include "Vocab.h"
+#include "Ngram.h"
+#include "hg.h"
+#include "stringlib.h"
+
+#ifdef HAVE_RANDLM
+#include "RandLM.h"
+#endif
+
+using namespace std;
+
+namespace NgramCache {
+  struct Cache {
+    map<WordID, Cache> tree;
+    float prob;
+    Cache() : prob() {}
+  };
+  static Cache cache_;
+  void Clear() { cache_.tree.clear(); }
+}
+
+struct LMClient {
+
+  LMClient(const char* host) : port(6666) {
+    strcpy(request_buffer, "prob ");
+    s = const_cast<char*>(strchr(host, ':'));  // TODO fix const_cast
+    if (s != NULL) {
+      *s = '\0';
+      ++s;
+      port = atoi(s);
+    }
+    sock = socket(AF_INET, SOCK_STREAM, 0);
+    hp = gethostbyname(host);
+    if (hp == NULL) {
+      cerr << "unknown host " << host << endl;
+      abort();
+    }
+    bzero((char *)&server, sizeof(server));
+    bcopy(hp->h_addr, (char *)&server.sin_addr, hp->h_length);
+    server.sin_family = hp->h_addrtype;
+    server.sin_port = htons(port);
+
+    int errors = 0;
+    while (connect(sock, (struct sockaddr *)&server, sizeof(server)) < 0) {
+      cerr << "Error: connect()\n";
+      sleep(1);
+      errors++;
+      if (errors > 3) exit(1);
+    }
+    cerr << "Connected to LM on " << host << " on port " << port << endl;
+  }
+
+  float wordProb(int word, int* context) {
+    NgramCache::Cache* cur = &NgramCache::cache_;
+    int i = 0;
+    while (context[i] > 0) {
+      cur = &cur->tree[context[i++]];
+    }
+    cur = &cur->tree[word];
+    if (cur->prob) { return cur->prob; }
+
+    i = 0;
+    int pos = TD::AppendString(word, 5, 16000, request_buffer);
+    while (context[i] > 0) {
+      assert(pos < 15995);
+      request_buffer[pos] = ' ';
+      ++pos;
+      pos = TD::AppendString(context[i], pos, 16000, request_buffer);
+      ++i;
+    }
+    assert(pos < 15999);
+    request_buffer[pos] = '\n';
+    ++pos;
+    request_buffer[pos] = 0;
+    write(sock, request_buffer, pos);
+    int r = read(sock, res, 6);
+    int errors = 0;
+    int cnt = 0;
+    while (1) {
+      if (r < 0) {
+        errors++; sleep(1);
+        cerr << "Error: read()\n";
+        if (errors > 5) exit(1);
+      } else if (r==0 || res[cnt] == '\n') { break; }
+      else {
+        cnt += r;
+        if (cnt==6) break;
+        read(sock, &res[cnt], 6-cnt);
+      }
+    }
+    cur->prob = *reinterpret_cast<float*>(res);
+    return cur->prob;
+  }
+
+ private:
+  int sock, port;
+  char *s;
+  struct hostent *hp;
+  struct sockaddr_in server;
+  char res[8];
+  char request_buffer[16000];
+};
+
+class LanguageModelImpl {
+ public:
+  explicit LanguageModelImpl(int order) :
+      ngram_(*TD::dict_, order), buffer_(), order_(order), state_size_(OrderToStateSize(order) - 1),
+      floor_(-100.0),
+      client_(),
+      kSTART(TD::Convert("<s>")),
+      kSTOP(TD::Convert("</s>")),
+      kUNKNOWN(TD::Convert("<unk>")),
+      kNONE(-1),
+      kSTAR(TD::Convert("<{STAR}>")) {}
+
+  LanguageModelImpl(int order, const string& f) :
+      ngram_(*TD::dict_, order), buffer_(), order_(order), state_size_(OrderToStateSize(order) - 1),
+      floor_(-100.0),
+      client_(NULL),
+      kSTART(TD::Convert("<s>")),
+      kSTOP(TD::Convert("</s>")),
+      kUNKNOWN(TD::Convert("<unk>")),
+      kNONE(-1),
+      kSTAR(TD::Convert("<{STAR}>")) {
+    if (f.find("lm://") == 0) {
+      client_ = new LMClient(f.substr(5).c_str());
+    } else {
+      File file(f.c_str(), "r", 0);
+      assert(file);
+      cerr << "Reading " << order_ << "-gram LM from " << f << endl;
+      ngram_.read(file, false);
+    }
+  }
+
+  virtual ~LanguageModelImpl() {
+    delete client_;
+  }
+
+  inline int StateSize(const void* state) const {
+    return *(static_cast<const char*>(state) + state_size_);
+  }
+
+  inline void SetStateSize(int size, void* state) const {
+    *(static_cast<char*>(state) + state_size_) = size;
+  }
+
+  virtual double WordProb(int word, int* context) {
+    return client_ ?
+          client_->wordProb(word, context)
+        : ngram_.wordProb(word, (VocabIndex*)context);
+  }
+
+  inline double LookupProbForBufferContents(int i) {
+//    int k = i; cerr << "P("; while(buffer_[k] > 0) { std::cerr << TD::Convert(buffer_[k++]) << " "; }
+    double p = WordProb(buffer_[i], &buffer_[i+1]);
+    if (p < floor_) p = floor_;
+//    cerr << ")=" << p << endl;
+    return p;
+  }
+
+  string DebugStateToString(const void* state) const {
+    int len = StateSize(state);
+    const int* astate = reinterpret_cast<const int*>(state);
+    string res = "[";
+    for (int i = 0; i < len; ++i) {
+      res += " ";
+      res += TD::Convert(astate[i]);
+    }
+    res += " ]";
+    return res;
+  }
+
+  inline double ProbNoRemnant(int i, int len) {
+    int edge = len;
+    bool flag = true;
+    double sum = 0.0;
+    while (i >= 0) {
+      if (buffer_[i] == kSTAR) {
+        edge = i;
+        flag = false;
+      } else if (buffer_[i] <= 0) {
+        edge = i;
+        flag = true;
+      } else {
+        if ((edge-i >= order_) || (flag && !(i == (len-1) && buffer_[i] == kSTART)))
+          sum += LookupProbForBufferContents(i);
+      }
+      --i;
+    }
+    return sum;
+  }
+
+  double EstimateProb(const vector<WordID>& phrase) {
+    int len = phrase.size();
+    buffer_.resize(len + 1);
+    buffer_[len] = kNONE;
+    int i = len - 1;
+    for (int j = 0; j < len; ++j,--i)
+      buffer_[i] = phrase[j];
+    return ProbNoRemnant(len - 1, len);
+  }
+
+  double EstimateProb(const void* state) {
+    int len = StateSize(state);
+    // cerr << "residual len: " << len << endl;
+    buffer_.resize(len + 1);
+    buffer_[len] = kNONE;
+    const int* astate = reinterpret_cast<const int*>(state);
+    int i = len - 1;
+    for (int j = 0; j < len; ++j,--i)
+      buffer_[i] = astate[j];
+    return ProbNoRemnant(len - 1, len);
+  }
+
+  double FinalTraversalCost(const void* state) {
+    int slen = StateSize(state);
+    int len = slen + 2;
+    // cerr << "residual len: " << len << endl;
+    buffer_.resize(len + 1);
+    buffer_[len] = kNONE;
+    buffer_[len-1] = kSTART;
+    const int* astate = reinterpret_cast<const int*>(state);
+    int i = len - 2;
+    for (int j = 0; j < slen; ++j,--i)
+      buffer_[i] = astate[j];
+    buffer_[i] = kSTOP;
+    assert(i == 0);
+    return ProbNoRemnant(len - 1, len);
+  }
+
+  double LookupWords(const TRule& rule, const vector<const void*>& ant_states, void* vstate) {
+    int len = rule.ELength() - rule.Arity();
+    for (int i = 0; i < ant_states.size(); ++i)
+      len += StateSize(ant_states[i]);
+    buffer_.resize(len + 1);
+    buffer_[len] = kNONE;
+    int i = len - 1;
+    const vector<WordID>& e = rule.e();
+    for (int j = 0; j < e.size(); ++j) {
+      if (e[j] < 1) {
+        const int* astate = reinterpret_cast<const int*>(ant_states[-e[j]]);
+        int slen = StateSize(astate);
+        for (int k = 0; k < slen; ++k)
+          buffer_[i--] = astate[k];
+      } else {
+        buffer_[i--] = e[j];
+      }
+    }
+
+    double sum = 0.0;
+    int* remnant = reinterpret_cast<int*>(vstate);
+    int j = 0;
+    i = len - 1;
+    int edge = len;
+
+    while (i >= 0) {
+      if (buffer_[i] == kSTAR) {
+        edge = i;
+      } else if (edge-i >= order_) {
+        sum += LookupProbForBufferContents(i);
+      } else if (edge == len && remnant) {
+        remnant[j++] = buffer_[i];
+      }
+      --i;
+    }
+    if (!remnant) return sum;
+
+    if (edge != len || len >= order_) {
+      remnant[j++] = kSTAR;
+      if (order_-1 < edge) edge = order_-1;
+      for (int i = edge-1; i >= 0; --i)
+        remnant[j++] = buffer_[i];
+    }
+
+    SetStateSize(j, vstate);
+    return sum;
+  }
+
+  static int OrderToStateSize(int order) {
+    return ((order-1) * 2 + 1) * sizeof(WordID) + 1;
+  }
+
+ protected:
+  Ngram ngram_;
+  vector<WordID> buffer_;
+  const int order_;
+  const int state_size_;
+  const double floor_;
+ private:
+  LMClient* client_;
+
+ public:
+  const WordID kSTART;
+  const WordID kSTOP;
+  const WordID kUNKNOWN;
+  const WordID kNONE;
+  const WordID kSTAR;
+};
+
+LanguageModel::LanguageModel(const string& param) :
+    fid_(FD::Convert("LanguageModel")) {
+  vector<string> argv;
+  int argc = SplitOnWhitespace(param, &argv);
+  int order = 3;
+  // TODO add support for -n FeatureName
+  string filename;
+  if (argc < 1) { cerr << "LanguageModel requires a filename, minimally!\n"; abort(); }
+  else if (argc == 1) { filename = argv[0]; }
+  else if (argc == 2 || argc > 3) { cerr << "Don't understand 'LanguageModel " << param << "'\n"; }
+  else if (argc == 3) {
+    if (argv[0] == "-o") {
+      order = atoi(argv[1].c_str());
+      filename = argv[2];
+    } else if (argv[1] == "-o") {
+      order = atoi(argv[2].c_str());
+      filename = argv[0];
+    }
+  }
+  SetStateSize(LanguageModelImpl::OrderToStateSize(order));
+  pimpl_ = new LanguageModelImpl(order, filename);
+}
+
+LanguageModel::~LanguageModel() {
+  delete pimpl_;
+}
+
+string LanguageModel::DebugStateToString(const void* state) const{
+  return pimpl_->DebugStateToString(state);
+}
+
+void LanguageModel::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                          const Hypergraph::Edge& edge,
+                                          const vector<const void*>& ant_states,
+                                          SparseVector<double>* features,
+                                          SparseVector<double>* estimated_features,
+                                          void* state) const {
+  (void) smeta;
+  features->set_value(fid_, pimpl_->LookupWords(*edge.rule_, ant_states, state));
+  estimated_features->set_value(fid_, pimpl_->EstimateProb(state));
+}
+
+void LanguageModel::FinalTraversalFeatures(const void* ant_state,
+                                           SparseVector<double>* features) const {
+  features->set_value(fid_, pimpl_->FinalTraversalCost(ant_state));
+}
+
+#ifdef HAVE_RANDLM
+struct RandLMImpl : public LanguageModelImpl {
+  RandLMImpl(int order, randlm::RandLM* rlm) :
+      LanguageModelImpl(order),
+      rlm_(rlm),
+      oov_(rlm->getWordID(rlm->getOOV())),
+      rb_(1000, oov_) {
+    map<int, randlm::WordID> map_cdec2randlm;
+    int max_wordid = 0;
+    for(map<randlm::Word, randlm::WordID>::const_iterator it = rlm->vocabStart();
+        it != rlm->vocabEnd(); ++it) {
+      const int cur = TD::Convert(it->first);
+      map_cdec2randlm[TD::Convert(it->first)] = it->second;
+      if (cur > max_wordid) max_wordid = cur;
+    }
+    cdec2randlm_.resize(max_wordid + 1, oov_);
+    for (map<int, randlm::WordID>::iterator it = map_cdec2randlm.begin();
+         it != map_cdec2randlm.end(); ++it)
+      cdec2randlm_[it->first] = it->second;
+    map_cdec2randlm.clear();
+  }
+
+  inline randlm::WordID Convert2RandLM(int w) {
+    return (w < cdec2randlm_.size() ? cdec2randlm_[w] : oov_);
+  }
+
+  virtual double WordProb(int word, int* context) {
+    int i = order_;
+    int c = 1;
+    rb_[i] = Convert2RandLM(word);
+    while (i > 1 && *context > 0) {
+      --i;
+      rb_[i] = Convert2RandLM(*context);
+      ++context;
+      ++c;
+    }
+    const void* finalState = 0;
+    int found;
+    //cerr << "I = " << i << endl;
+    return rlm_->getProb(&rb_[i], c, &found, &finalState);
+  }
+ private:
+  boost::shared_ptr<randlm::RandLM> rlm_;
+  randlm::WordID oov_;
+  vector<randlm::WordID> cdec2randlm_;
+  vector<randlm::WordID> rb_;
+};
+
+LanguageModelRandLM::LanguageModelRandLM(const string& param) :
+    fid_(FD::Convert("RandLM")) {
+  vector<string> argv;
+  int argc = SplitOnWhitespace(param, &argv);
+  int order = 3;
+  // TODO add support for -n FeatureName
+  string filename;
+  if (argc < 1) { cerr << "RandLM requires a filename, minimally!\n"; abort(); }
+  else if (argc == 1) { filename = argv[0]; }
+  else if (argc == 2 || argc > 3) { cerr << "Don't understand 'RandLM " << param << "'\n"; }
+  else if (argc == 3) {
+    if (argv[0] == "-o") {
+      order = atoi(argv[1].c_str());
+      filename = argv[2];
+    } else if (argv[1] == "-o") {
+      order = atoi(argv[2].c_str());
+      filename = argv[0];
+    }
+  }
+  SetStateSize(LanguageModelImpl::OrderToStateSize(order));
+  int cache_MB = 200; // increase cache size
+  randlm::RandLM* rlm = randlm::RandLM::initRandLM(filename, order, cache_MB);
+  assert(rlm != NULL);
+  pimpl_ = new RandLMImpl(order, rlm);
+}
+
+LanguageModelRandLM::~LanguageModelRandLM() {
+  delete pimpl_;
+}
+
+void LanguageModelRandLM::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                          const Hypergraph::Edge& edge,
+                                          const vector<const void*>& ant_states,
+                                          SparseVector<double>* features,
+                                          SparseVector<double>* estimated_features,
+                                          void* state) const {
+  (void) smeta;
+  features->set_value(fid_, pimpl_->LookupWords(*edge.rule_, ant_states, state));
+  estimated_features->set_value(fid_, pimpl_->EstimateProb(state));
+}
+
+void LanguageModelRandLM::FinalTraversalFeatures(const void* ant_state,
+                                           SparseVector<double>* features) const {
+  features->set_value(fid_, pimpl_->FinalTraversalCost(ant_state));
+}
+
+#endif
+
diff --git a/decoder/ff_lm.h b/decoder/ff_lm.h
new file mode 100644
index 00000000..45fc1da7
--- /dev/null
+++ b/decoder/ff_lm.h
@@ -0,0 +1,55 @@
+#ifndef _LM_FF_H_
+#define _LM_FF_H_
+
+#include <vector>
+#include <string>
+
+#include "hg.h"
+#include "ff.h"
+#include "config.h"
+
+class LanguageModelImpl;
+
+class LanguageModel : public FeatureFunction {
+ public:
+  // param = "filename.lm [-o n]"
+  LanguageModel(const std::string& param);
+  ~LanguageModel();
+  virtual void FinalTraversalFeatures(const void* context,
+                                      SparseVector<double>* features) const;
+  std::string DebugStateToString(const void* state) const;
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  const int fid_;
+  mutable LanguageModelImpl* pimpl_;
+};
+
+#ifdef HAVE_RANDLM
+class LanguageModelRandLM : public FeatureFunction {
+ public:
+  // param = "filename.lm [-o n]"
+  LanguageModelRandLM(const std::string& param);
+  ~LanguageModelRandLM();
+  virtual void FinalTraversalFeatures(const void* context,
+                                      SparseVector<double>* features) const;
+  std::string DebugStateToString(const void* state) const;
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  const int fid_;
+  mutable LanguageModelImpl* pimpl_;
+};
+#endif
+
+#endif
diff --git a/decoder/ff_tagger.cc b/decoder/ff_tagger.cc
new file mode 100644
index 00000000..7a9d1def
--- /dev/null
+++ b/decoder/ff_tagger.cc
@@ -0,0 +1,96 @@
+#include "ff_tagger.h"
+
+#include "tdict.h"
+#include "sentence_metadata.h"
+
+#include <sstream>
+
+using namespace std;
+
+Tagger_BigramIdentity::Tagger_BigramIdentity(const std::string& param) :
+  FeatureFunction(sizeof(WordID)) {}
+
+void Tagger_BigramIdentity::FireFeature(const WordID& left,
+                                 const WordID& right,
+                                 SparseVector<double>* features) const {
+  int& fid = fmap_[left][right];
+  if (!fid) {
+    ostringstream os;
+    if (right == 0) {
+      os << "Uni:" << TD::Convert(left);
+    } else {
+      os << "Bi:";
+      if (left < 0) { os << "BOS"; } else { os << TD::Convert(left); }
+      os << '_';
+      if (right < 0) { os << "EOS"; } else { os << TD::Convert(right); }
+    }
+    fid = FD::Convert(os.str());
+  }
+  features->set_value(fid, 1.0);
+}
+
+void Tagger_BigramIdentity::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const {
+  WordID& out_context = *static_cast<WordID*>(context);
+  const int arity = edge.Arity();
+  if (arity == 0) {
+    out_context = edge.rule_->e_[0];
+    FireFeature(out_context, 0, features);
+  } else if (arity == 2) {
+    WordID left = *static_cast<const WordID*>(ant_contexts[0]);
+    WordID right = *static_cast<const WordID*>(ant_contexts[1]);
+    if (edge.i_ == 0 && edge.j_ == 2)
+      FireFeature(-1, left, features);
+    FireFeature(left, right, features);
+    if (edge.i_ == 0 && edge.j_ == smeta.GetSourceLength())
+      FireFeature(right, -1, features);
+    out_context = right;
+  }
+}
+
+LexicalPairIdentity::LexicalPairIdentity(const std::string& param) {}
+
+void LexicalPairIdentity::FireFeature(WordID src,
+                                 WordID trg,
+                                 SparseVector<double>* features) const {
+  int& fid = fmap_[src][trg];
+  if (!fid) {
+    static map<WordID, WordID> escape;
+    if (escape.empty()) {
+      escape[TD::Convert("=")] = TD::Convert("__EQ");
+      escape[TD::Convert(";")] = TD::Convert("__SC");
+      escape[TD::Convert(",")] = TD::Convert("__CO");
+    }
+    if (escape.count(src)) src = escape[src];
+    if (escape.count(trg)) trg = escape[trg];
+    ostringstream os;
+    os << "Id:" << TD::Convert(src) << ':' << TD::Convert(trg);
+    fid = FD::Convert(os.str());
+  }
+  features->set_value(fid, 1.0);
+}
+
+void LexicalPairIdentity::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const {
+  const vector<WordID>& ew = edge.rule_->e_;
+  const vector<WordID>& fw = edge.rule_->f_;
+  for (int i = 0; i < ew.size(); ++i) {
+    const WordID& e = ew[i];
+    if (e <= 0) continue;
+    for (int j = 0; j < fw.size(); ++j) {
+      const WordID& f = fw[j];
+      if (f <= 0) continue;
+      FireFeature(f, e, features);
+    }
+  }
+}
+
+
diff --git a/decoder/ff_tagger.h b/decoder/ff_tagger.h
new file mode 100644
index 00000000..41c3ee5b
--- /dev/null
+++ b/decoder/ff_tagger.h
@@ -0,0 +1,51 @@
+#ifndef _FF_TAGGER_H_
+#define _FF_TAGGER_H_
+
+#include <map>
+#include "ff.h"
+
+typedef std::map<WordID, int> Class2FID;
+typedef std::map<WordID, Class2FID> Class2Class2FID;
+
+// the reason this is a "tagger" feature is that it assumes that
+// the sequence unfolds from left to right, which means it doesn't
+// have to split states based on left context.
+// fires unigram features as well
+class Tagger_BigramIdentity : public FeatureFunction {
+ public:
+  Tagger_BigramIdentity(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const;
+ private:
+  void FireFeature(const WordID& left,
+                   const WordID& right,
+                   SparseVector<double>* features) const;
+  mutable Class2Class2FID fmap_;
+};
+
+// for each pair of symbols cooccuring in a lexicalized rule, fire
+// a feature (mostly used for tagging, but could be used for any model)
+class LexicalPairIdentity : public FeatureFunction {
+ public:
+  LexicalPairIdentity(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const;
+ private:
+  void FireFeature(WordID src,
+                   WordID trg,
+                   SparseVector<double>* features) const;
+  mutable Class2Class2FID fmap_;
+};
+
+
+#endif
diff --git a/decoder/ff_test.cc b/decoder/ff_test.cc
new file mode 100644
index 00000000..9e640517
--- /dev/null
+++ b/decoder/ff_test.cc
@@ -0,0 +1,64 @@
+#include <cassert>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <gtest/gtest.h>
+#include "hg.h"
+#include "ff_lm.h"
+#include "ff.h"
+#include "trule.h"
+#include "sentence_metadata.h"
+
+using namespace std;
+
+LanguageModel* lm_ = NULL;
+LanguageModel* lm3_ = NULL;
+
+class FFTest : public testing::Test {
+ public:
+  FFTest() : smeta(0,Lattice()) {
+    if (!lm_) {
+      static LanguageModel slm("-o 2 ./test_data/test_2gram.lm.gz");
+      lm_ = &slm;
+      static LanguageModel slm3("./test_data/dummy.3gram.lm -o 3");
+      lm3_ = &slm3;
+    }
+  }
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+  SentenceMetadata smeta;
+};
+       
+TEST_F(FFTest, LM3) {
+  int x = lm3_->NumBytesContext();
+  Hypergraph::Edge edge1;
+  edge1.rule_.reset(new TRule("[X] ||| x y ||| one ||| 1.0 -2.4 3.0"));
+  Hypergraph::Edge edge2;
+  edge2.rule_.reset(new TRule("[X] ||| [X,1] a ||| [X,1] two ||| 1.0 -2.4 3.0"));
+  Hypergraph::Edge edge3;
+  edge3.rule_.reset(new TRule("[X] ||| [X,1] a ||| zero [X,1] two ||| 1.0 -2.4 3.0"));
+  vector<const void*> ants1;
+  string state(x, '\0');
+  SparseVector<double> feats;
+  SparseVector<double> est;
+  lm3_->TraversalFeatures(smeta, edge1, ants1, &feats, &est, (void *)&state[0]);
+  cerr << "returned " << feats << endl;
+  cerr << edge1.feature_values_ << endl;
+  cerr << lm3_->DebugStateToString((const void*)&state[0]) << endl;
+  EXPECT_EQ("[ one ]", lm3_->DebugStateToString((const void*)&state[0]));
+  ants1.push_back((const void*)&state[0]);
+  string state2(x, '\0');
+  lm3_->TraversalFeatures(smeta, edge2, ants1, &feats, &est, (void *)&state2[0]);
+  cerr << lm3_->DebugStateToString((const void*)&state2[0]) << endl;
+  EXPECT_EQ("[ one two ]", lm3_->DebugStateToString((const void*)&state2[0]));
+  string state3(x, '\0');
+  lm3_->TraversalFeatures(smeta, edge3, ants1, &feats, &est, (void *)&state3[0]);
+  cerr << lm3_->DebugStateToString((const void*)&state3[0]) << endl;
+  EXPECT_EQ("[ zero one <{STAR}> one two ]", lm3_->DebugStateToString((const void*)&state3[0]));
+}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/decoder/ff_wordalign.cc b/decoder/ff_wordalign.cc
new file mode 100644
index 00000000..669aa530
--- /dev/null
+++ b/decoder/ff_wordalign.cc
@@ -0,0 +1,445 @@
+#include "ff_wordalign.h"
+
+#include <set>
+#include <sstream>
+#include <string>
+#include <cmath>
+
+#include "stringlib.h"
+#include "sentence_metadata.h"
+#include "hg.h"
+#include "fdict.h"
+#include "aligner.h"
+#include "tdict.h"   // Blunsom hack
+#include "filelib.h" // Blunsom hack
+
+static const int MAX_SENTENCE_SIZE = 100;
+
+using namespace std;
+
+Model2BinaryFeatures::Model2BinaryFeatures(const string& param) :
+    fids_(boost::extents[MAX_SENTENCE_SIZE][MAX_SENTENCE_SIZE][MAX_SENTENCE_SIZE]) {
+  for (int i = 1; i < MAX_SENTENCE_SIZE; ++i) {
+    for (int j = 0; j < i; ++j) {
+      for (int k = 0; k < MAX_SENTENCE_SIZE; ++k) {
+        int& val = fids_[i][j][k];
+        val = -1;
+        if (j < i) {
+          ostringstream os;
+          os << "M2FL:" << i << ":TI:" << k << "_SI:" << j;
+          val = FD::Convert(os.str());
+        }
+      }
+    }
+  }
+}
+
+void Model2BinaryFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                                 const Hypergraph::Edge& edge,
+                                                 const vector<const void*>& ant_states,
+                                                 SparseVector<double>* features,
+                                                 SparseVector<double>* estimated_features,
+                                                 void* state) const {
+  // if the source word is either null or the generated word
+  // has no position in the reference
+  if (edge.i_ == -1 || edge.prev_i_ == -1)
+    return;
+
+  assert(smeta.GetTargetLength() > 0);
+  const int fid = fids_[smeta.GetSourceLength()][edge.i_][edge.prev_i_];
+  features->set_value(fid, 1.0);
+//  cerr << f_len_ << " " << e_len_ << " [" << edge.i_ << "," << edge.j_ << "|" << edge.prev_i_ << "," << edge.prev_j_ << "]\t" << edge.rule_->AsString() << "\tVAL=" << val << endl;
+}
+
+
+RelativeSentencePosition::RelativeSentencePosition(const string& param) :
+    fid_(FD::Convert("RelativeSentencePosition")) {
+  if (!param.empty()) {
+    cerr << "  Loading word classes from " << param << endl;
+    condition_on_fclass_ = true;
+    ReadFile rf(param);
+    istream& in = *rf.stream();
+    set<WordID> classes;
+    while(in) {
+      string line;
+      getline(in, line);
+      if (line.empty()) continue;
+      vector<WordID> v;
+      TD::ConvertSentence(line, &v);
+      pos_.push_back(v);
+      for (int i = 0; i < v.size(); ++i)
+        classes.insert(v[i]);
+      for (set<WordID>::iterator i = classes.begin(); i != classes.end(); ++i) {
+        ostringstream os;
+        os << "RelPos_FC:" << TD::Convert(*i);
+        fids_[*i] = FD::Convert(os.str());
+      }
+    }
+  } else {
+    condition_on_fclass_ = false;
+  }
+}
+
+void RelativeSentencePosition::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                                     const Hypergraph::Edge& edge,
+                                                     const vector<const void*>& ant_states,
+                                                     SparseVector<double>* features,
+                                                     SparseVector<double>* estimated_features,
+                                                     void* state) const {
+  // if the source word is either null or the generated word
+  // has no position in the reference
+  if (edge.i_ == -1 || edge.prev_i_ == -1)
+    return;
+
+  assert(smeta.GetTargetLength() > 0);
+  const double val = fabs(static_cast<double>(edge.i_) / smeta.GetSourceLength() -
+                          static_cast<double>(edge.prev_i_) / smeta.GetTargetLength());
+  features->set_value(fid_, val);
+  if (condition_on_fclass_) {
+    assert(smeta.GetSentenceID() < pos_.size());
+    const WordID cur_fclass = pos_[smeta.GetSentenceID()][edge.i_];
+    const int fid = fids_.find(cur_fclass)->second;
+    features->set_value(fid, val);
+  }
+//  cerr << f_len_ << " " << e_len_ << " [" << edge.i_ << "," << edge.j_ << "|" << edge.prev_i_ << "," << edge.prev_j_ << "]\t" << edge.rule_->AsString() << "\tVAL=" << val << endl;
+}
+
+MarkovJumpFClass::MarkovJumpFClass(const string& param) :
+    FeatureFunction(1),
+    fids_(MAX_SENTENCE_SIZE) {
+  cerr << "    MarkovJumpFClass" << endl;
+  cerr << "Reading source POS tags from " << param << endl;
+  ReadFile rf(param);
+  istream& in = *rf.stream();
+  set<WordID> classes;
+  while(in) {
+    string line;
+    getline(in, line);
+    if (line.empty()) continue;
+    vector<WordID> v;
+    TD::ConvertSentence(line, &v);
+    pos_.push_back(v);
+    for (int i = 0; i < v.size(); ++i)
+      classes.insert(v[i]);
+  }
+  cerr << "  (" << pos_.size() << " lines)\n";
+  cerr << "  Classes: " << classes.size() << endl;
+  for (int ss = 1; ss < MAX_SENTENCE_SIZE; ++ss) {
+    map<WordID, map<int, int> >& cfids = fids_[ss];
+    for (set<WordID>::iterator i = classes.begin(); i != classes.end(); ++i) {
+      map<int, int> &fids = cfids[*i];
+      for (int j = -ss; j <= ss; ++j) {
+        ostringstream os;
+        os << "Jump_FL:" << ss << "_FC:" << TD::Convert(*i) << "_J:" << j;
+        fids[j] = FD::Convert(os.str());
+      }
+    }
+  }
+}
+
+void MarkovJumpFClass::FireFeature(const SentenceMetadata& smeta,
+                                   int prev_src_pos,
+                                   int cur_src_pos,
+                                   SparseVector<double>* features) const {
+  const int jumpsize = cur_src_pos - prev_src_pos;
+  assert(smeta.GetSentenceID() < pos_.size());
+  const WordID cur_fclass = pos_[smeta.GetSentenceID()][cur_src_pos];
+  const int fid = fids_[smeta.GetSourceLength()].find(cur_fclass)->second.find(jumpsize)->second;
+  features->set_value(fid, 1.0);
+}
+
+void MarkovJumpFClass::FinalTraversalFeatures(const void* context,
+                                      SparseVector<double>* features) const {
+  int left_index = *static_cast<const unsigned char*>(context);
+//  int right_index = cur_flen;
+  // TODO
+}
+
+void MarkovJumpFClass::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_states,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* state) const {
+  unsigned char& dpstate = *((unsigned char*)state);
+  if (edge.Arity() == 0) {
+    dpstate = static_cast<unsigned int>(edge.i_);
+  } else if (edge.Arity() == 1) {
+    dpstate = *((unsigned char*)ant_states[0]);
+  } else if (edge.Arity() == 2) {
+    int left_index = *((unsigned char*)ant_states[0]);
+    int right_index = *((unsigned char*)ant_states[1]);
+    if (right_index == -1)
+      dpstate = static_cast<unsigned int>(left_index);
+    else
+      dpstate = static_cast<unsigned int>(right_index);
+//    const WordID cur_fclass = pos_[smeta.GetSentenceID()][right_index];
+//    cerr << edge.i_ << "," << edge.j_ << ": fclass=" << TD::Convert(cur_fclass) << " j=" << jumpsize << endl;
+//    const int fid = fids_[smeta.GetSourceLength()].find(cur_fclass)->second.find(jumpsize)->second;
+//    features->set_value(fid, 1.0);
+    FireFeature(smeta, left_index, right_index, features);
+  }
+}
+
+//  std::vector<std::map<int, int> > flen2jump2fid_;
+MarkovJump::MarkovJump(const string& param) :
+    FeatureFunction(1),
+    fid_(FD::Convert("MarkovJump")),
+    binary_params_(false) {
+  cerr << "    MarkovJump";
+  vector<string> argv;
+  int argc = SplitOnWhitespace(param, &argv);
+  if (argc != 1 || !(argv[0] == "-b" || argv[0] == "+b")) {
+    cerr << "MarkovJump: expected parameters to be -b or +b\n";
+    exit(1);
+  }
+  binary_params_ = argv[0] == "+b";
+  if (binary_params_) {
+    flen2jump2fid_.resize(MAX_SENTENCE_SIZE);
+    for (int i = 1; i < MAX_SENTENCE_SIZE; ++i) {
+      map<int, int>& jump2fid = flen2jump2fid_[i];
+      for (int jump = -i; jump <= i; ++jump) {
+        ostringstream os;
+        os << "Jump:FLen:" << i << "_J:" << jump;
+        jump2fid[jump] = FD::Convert(os.str());
+      }
+    }
+  } else {
+    cerr << " (Blunsom & Cohn definition)";
+  }
+  cerr << endl;
+}
+
+// TODO handle NULLs according to Och 2000
+void MarkovJump::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                       const Hypergraph::Edge& edge,
+                                       const vector<const void*>& ant_states,
+                                       SparseVector<double>* features,
+                                       SparseVector<double>* estimated_features,
+                                       void* state) const {
+  unsigned char& dpstate = *((unsigned char*)state);
+  const int flen = smeta.GetSourceLength();
+  if (edge.Arity() == 0) {
+    dpstate = static_cast<unsigned int>(edge.i_);
+    if (edge.prev_i_ == 0) {
+      if (binary_params_) {
+        // NULL will be tricky
+        // TODO initial state distribution, not normal jumps
+        const int fid = flen2jump2fid_[flen].find(edge.i_ + 1)->second;
+        features->set_value(fid, 1.0);
+      }
+    } else if (edge.prev_i_ == smeta.GetTargetLength() - 1) {
+        // NULL will be tricky
+      if (binary_params_) {
+        int jumpsize = flen - edge.i_;
+        const int fid = flen2jump2fid_[flen].find(jumpsize)->second;
+        features->set_value(fid, 1.0);
+      }
+    }
+  } else if (edge.Arity() == 1) {
+    dpstate = *((unsigned char*)ant_states[0]);
+  } else if (edge.Arity() == 2) {
+    int left_index = *((unsigned char*)ant_states[0]);
+    int right_index = *((unsigned char*)ant_states[1]);
+    if (right_index == -1)
+      dpstate = static_cast<unsigned int>(left_index);
+    else
+      dpstate = static_cast<unsigned int>(right_index);
+    const int jumpsize = right_index - left_index;
+
+    if (binary_params_) {
+      const int fid = flen2jump2fid_[flen].find(jumpsize)->second;
+      features->set_value(fid, 1.0);
+    } else {
+      features->set_value(fid_, fabs(jumpsize - 1));  // Blunsom and Cohn def
+    }
+  } else {
+    assert(!"something really unexpected is happening");
+  }
+}
+
+// state: POS of src word used, number of trg words generated
+SourcePOSBigram::SourcePOSBigram(const std::string& param) :
+    FeatureFunction(sizeof(WordID) + sizeof(int)) {
+  cerr << "Reading source POS tags from " << param << endl;
+  ReadFile rf(param);
+  istream& in = *rf.stream();
+  while(in) {
+    string line;
+    getline(in, line);
+    if (line.empty()) continue;
+    vector<WordID> v;
+    TD::ConvertSentence(line, &v);
+    pos_.push_back(v);
+  }
+  cerr << "  (" << pos_.size() << " lines)\n";
+}
+
+void SourcePOSBigram::FinalTraversalFeatures(const void* context,
+                                      SparseVector<double>* features) const {
+  WordID left = *static_cast<const WordID*>(context);
+  int left_wc = *(static_cast<const int*>(context) + 1);
+  if (left_wc == 1)
+    FireFeature(-1, left, features);
+  FireFeature(left, -1, features);
+}
+
+void SourcePOSBigram::FireFeature(WordID left,
+                   WordID right,
+                   SparseVector<double>* features) const {
+  int& fid = fmap_[left][right];
+  if (!fid) {
+    ostringstream os;
+    os << "SP:";
+    if (left < 0) { os << "BOS"; } else { os << TD::Convert(left); }
+    os << '_';
+    if (right < 0) { os << "EOS"; } else { os << TD::Convert(right); }
+    fid = FD::Convert(os.str());
+    if (fid == 0) fid = -1;
+  }
+  if (fid < 0) return;
+  features->set_value(fid, 1.0);
+}
+
+void SourcePOSBigram::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const {
+  WordID& out_context = *static_cast<WordID*>(context);
+  int& out_word_count = *(static_cast<int*>(context) + 1);
+  const int arity = edge.Arity();
+  if (arity == 0) {
+    assert(smeta.GetSentenceID() < pos_.size());
+    const vector<WordID>& pos_sent = pos_[smeta.GetSentenceID()];
+    assert(edge.i_ < pos_sent.size());
+    out_context = pos_sent[edge.i_];
+    out_word_count = edge.rule_->EWords();
+    assert(out_word_count == 1); // this is only defined for lex translation!
+    // revisit this if you want to translate into null words
+  } else if (arity == 2) {
+    WordID left = *static_cast<const WordID*>(ant_contexts[0]);
+    WordID right = *static_cast<const WordID*>(ant_contexts[1]);
+    int left_wc = *(static_cast<const int*>(ant_contexts[0]) + 1);
+    int right_wc = *(static_cast<const int*>(ant_contexts[0]) + 1);
+    if (left_wc == 1 && right_wc == 1)
+      FireFeature(-1, left, features);
+    FireFeature(left, right, features);
+    out_word_count = left_wc + right_wc;
+    out_context = right;
+  }
+}
+
+AlignerResults::AlignerResults(const std::string& param) :
+    cur_sent_(-1),
+    cur_grid_(NULL) {
+  vector<string> argv;
+  int argc = SplitOnWhitespace(param, &argv);
+  if (argc != 2) {
+    cerr << "Required format: AlignerResults [FeatureName] [file.pharaoh]\n";
+    exit(1);
+  }
+  cerr << "  feature: " << argv[0] << "\talignments: " << argv[1] << endl;
+  fid_ = FD::Convert(argv[0]);
+  ReadFile rf(argv[1]);
+  istream& in = *rf.stream(); int lc = 0;
+  while(in) {
+    string line;
+    getline(in, line);
+    if (!in) break; 
+    ++lc;
+    is_aligned_.push_back(AlignerTools::ReadPharaohAlignmentGrid(line));
+  }
+  cerr << "  Loaded " << lc << " refs\n";
+}
+
+void AlignerResults::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                           const Hypergraph::Edge& edge,
+                                           const vector<const void*>& ant_states,
+                                           SparseVector<double>* features,
+                                           SparseVector<double>* estimated_features,
+                                           void* state) const {
+  if (edge.i_ == -1 || edge.prev_i_ == -1)
+    return;
+
+  if (cur_sent_ != smeta.GetSentenceID()) {
+    assert(smeta.HasReference());
+    cur_sent_ = smeta.GetSentenceID();
+    assert(cur_sent_ < is_aligned_.size());
+    cur_grid_ = is_aligned_[cur_sent_].get();
+  }
+
+  //cerr << edge.rule_->AsString() << endl;
+
+  int j = edge.i_;        // source side (f)
+  int i = edge.prev_i_;   // target side (e)
+  if (j < cur_grid_->height() && i < cur_grid_->width() && (*cur_grid_)(i, j)) {
+//    if (edge.rule_->e_[0] == smeta.GetReference()[i][0].label) {
+      features->set_value(fid_, 1.0);
+//      cerr << edge.rule_->AsString() << "   (" << i << "," << j << ")\n";
+//    }
+  }
+}
+
+BlunsomSynchronousParseHack::BlunsomSynchronousParseHack(const string& param) :
+  FeatureFunction((100 / 8) + 1), fid_(FD::Convert("NotRef")), cur_sent_(-1) {
+  ReadFile rf(param);
+  istream& in = *rf.stream(); int lc = 0;
+  while(in) {
+    string line;
+    getline(in, line);
+    if (!in) break; 
+    ++lc;
+    refs_.push_back(vector<WordID>());
+    TD::ConvertSentence(line, &refs_.back());
+  }
+  cerr << "  Loaded " << lc << " refs\n";
+}
+
+void BlunsomSynchronousParseHack::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                           const Hypergraph::Edge& edge,
+                                           const vector<const void*>& ant_states,
+                                           SparseVector<double>* features,
+                                           SparseVector<double>* estimated_features,
+                                           void* state) const {
+  if (cur_sent_ != smeta.GetSentenceID()) {
+    // assert(smeta.HasReference());
+    cur_sent_ = smeta.GetSentenceID();
+    assert(cur_sent_ < refs_.size());
+    cur_ref_ = &refs_[cur_sent_];
+    cur_map_.clear();
+    for (int i = 0; i < cur_ref_->size(); ++i) {
+      vector<WordID> phrase;
+      for (int j = i; j < cur_ref_->size(); ++j) {
+        phrase.push_back((*cur_ref_)[j]);
+        cur_map_[phrase] = i;
+      }
+    }
+  }
+  //cerr << edge.rule_->AsString() << endl;
+  for (int i = 0; i < ant_states.size(); ++i) {
+    if (DoesNotBelong(ant_states[i])) {
+      //cerr << "  ant " << i << " does not belong\n";
+      return;
+    }
+  }
+  vector<vector<WordID> > ants(ant_states.size());
+  vector<const vector<WordID>* > pants(ant_states.size());
+  for (int i = 0; i < ant_states.size(); ++i) {
+    AppendAntecedentString(ant_states[i], &ants[i]);
+    //cerr << "  ant[" << i << "]: " << ((int)*(static_cast<const unsigned char*>(ant_states[i]))) << " " << TD::GetString(ants[i]) << endl;
+    pants[i] = &ants[i];
+  }
+  vector<WordID> yield;
+  edge.rule_->ESubstitute(pants, &yield);
+  //cerr << "YIELD: " << TD::GetString(yield) << endl;
+  Vec2Int::iterator it = cur_map_.find(yield);
+  if (it == cur_map_.end()) {
+    features->set_value(fid_, 1);
+    //cerr << "  BAD!\n";
+    return;
+  }
+  SetStateMask(it->second, it->second + yield.size(), state);
+}
+
diff --git a/decoder/ff_wordalign.h b/decoder/ff_wordalign.h
new file mode 100644
index 00000000..c44ad26b
--- /dev/null
+++ b/decoder/ff_wordalign.h
@@ -0,0 +1,196 @@
+#ifndef _FF_WORD_ALIGN_H_
+#define _FF_WORD_ALIGN_H_
+
+#include "ff.h"
+#include "array2d.h"
+
+#include <boost/multi_array.hpp>
+
+class RelativeSentencePosition : public FeatureFunction {
+ public:
+  RelativeSentencePosition(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  const int fid_;
+  bool condition_on_fclass_;
+  std::vector<std::vector<WordID> > pos_;
+  std::map<WordID, int> fids_;  // fclass -> fid
+};
+
+class Model2BinaryFeatures : public FeatureFunction {
+ public:
+  Model2BinaryFeatures(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  boost::multi_array<int, 3> fids_;
+};
+
+class MarkovJump : public FeatureFunction {
+ public:
+  MarkovJump(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  const int fid_;
+  bool binary_params_;
+  std::vector<std::map<int, int> > flen2jump2fid_;
+};
+
+class MarkovJumpFClass : public FeatureFunction {
+ public:
+  MarkovJumpFClass(const std::string& param);
+  virtual void FinalTraversalFeatures(const void* context,
+                                      SparseVector<double>* features) const;
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const;
+
+  void FireFeature(const SentenceMetadata& smeta,
+                   int prev_src_pos,
+                   int cur_src_pos,
+                   SparseVector<double>* features) const;
+
+ private:
+  std::vector<std::map<WordID, std::map<int, int> > > fids_;  // flen -> fclass -> jumpsize -> fid
+  std::vector<std::vector<WordID> > pos_;
+};
+
+typedef std::map<WordID, int> Class2FID;
+typedef std::map<WordID, Class2FID> Class2Class2FID;
+class SourcePOSBigram : public FeatureFunction {
+ public:
+  SourcePOSBigram(const std::string& param);
+  virtual void FinalTraversalFeatures(const void* context,
+                                      SparseVector<double>* features) const;
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const;
+ private:
+  void FireFeature(WordID src,
+                   WordID trg,
+                   SparseVector<double>* features) const;
+  mutable Class2Class2FID fmap_;
+  std::vector<std::vector<WordID> > pos_;
+};
+
+class AlignerResults : public FeatureFunction {
+ public:
+  AlignerResults(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  int fid_;
+  std::vector<boost::shared_ptr<Array2D<bool> > > is_aligned_;
+  mutable int cur_sent_;
+  const Array2D<bool> mutable* cur_grid_;
+};
+
+#include <tr1/unordered_map>
+#include <boost/functional/hash.hpp>
+#include <cassert>
+class BlunsomSynchronousParseHack : public FeatureFunction {
+ public:
+  BlunsomSynchronousParseHack(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  inline bool DoesNotBelong(const void* state) const {
+    for (int i = 0; i < NumBytesContext(); ++i) {
+      if (*(static_cast<const unsigned char*>(state) + i)) return false;
+    }
+    return true;
+  }
+
+  inline void AppendAntecedentString(const void* state, std::vector<WordID>* yield) const {
+    int i = 0;
+    int ind = 0;
+    while (i < NumBytesContext() && !(*(static_cast<const unsigned char*>(state) + i))) { ++i; ind += 8; }
+    // std::cerr << i << " " << NumBytesContext() << std::endl;
+    assert(i != NumBytesContext());
+    assert(ind < cur_ref_->size());
+    int cur = *(static_cast<const unsigned char*>(state) + i);
+    int comp = 1;
+    while (comp < 256 && (comp & cur) == 0) { comp <<= 1; ++ind; }
+    assert(ind < cur_ref_->size());
+    assert(comp < 256);
+    do {
+      assert(ind < cur_ref_->size());
+      yield->push_back((*cur_ref_)[ind]);
+      ++ind;
+      comp <<= 1;
+      if (comp == 256) {
+        comp = 1;
+        ++i;
+        cur = *(static_cast<const unsigned char*>(state) + i);
+      }
+    } while (comp & cur);
+  }
+
+  inline void SetStateMask(int start, int end, void* state) const {
+    assert((end / 8) < NumBytesContext());
+    int i = 0;
+    int comp = 1;
+    for (int j = 0; j < start; ++j) {
+      comp <<= 1;
+      if (comp == 256) {
+        ++i;
+        comp = 1;
+      }
+    }
+    //std::cerr << "SM: " << i << "\n";
+    for (int j = start; j < end; ++j) {
+      *(static_cast<unsigned char*>(state) + i) |= comp;
+      //std::cerr << "  " << comp << "\n";
+      comp <<= 1;
+      if (comp == 256) {
+        ++i;
+        comp = 1;
+      }
+    }
+    //std::cerr << "   MASK: " << ((int)*(static_cast<unsigned char*>(state))) << "\n";
+  }
+
+  const int fid_;
+  mutable int cur_sent_;
+  typedef std::tr1::unordered_map<std::vector<WordID>, int, boost::hash<std::vector<WordID> > > Vec2Int;
+  mutable Vec2Int cur_map_;
+  const std::vector<WordID> mutable * cur_ref_;
+  mutable std::vector<std::vector<WordID> > refs_;
+};
+
+#endif
diff --git a/decoder/filelib.cc b/decoder/filelib.cc
new file mode 100644
index 00000000..79ad2847
--- /dev/null
+++ b/decoder/filelib.cc
@@ -0,0 +1,22 @@
+#include "filelib.h"
+
+#include <unistd.h>
+#include <sys/stat.h>
+
+using namespace std;
+
+bool FileExists(const std::string& fn) {
+  struct stat info;
+  int s = stat(fn.c_str(), &info);
+  return (s==0);
+}
+
+bool DirectoryExists(const string& dir) {
+  if (access(dir.c_str(),0) == 0) {
+    struct stat status;
+    stat(dir.c_str(), &status);
+    if (status.st_mode & S_IFDIR) return true;
+  }
+  return false;
+}
+
diff --git a/decoder/filelib.h b/decoder/filelib.h
new file mode 100644
index 00000000..03c22b0d
--- /dev/null
+++ b/decoder/filelib.h
@@ -0,0 +1,70 @@
+#ifndef _FILELIB_H_
+#define _FILELIB_H_
+
+#include <cassert>
+#include <string>
+#include <iostream>
+#include <cstdlib>
+#include "gzstream.h"
+
+bool FileExists(const std::string& file_name);
+bool DirectoryExists(const std::string& dir_name);
+
+// reads from standard in if filename is -
+// uncompresses if file ends with .gz
+// otherwise, reads from a normal file
+class ReadFile {
+ public:
+  ReadFile(const std::string& filename) :
+    no_delete_on_exit_(filename == "-"),
+    in_(no_delete_on_exit_ ? static_cast<std::istream*>(&std::cin) :
+      (EndsWith(filename, ".gz") ?
+        static_cast<std::istream*>(new igzstream(filename.c_str())) :
+        static_cast<std::istream*>(new std::ifstream(filename.c_str())))) {
+    if (!no_delete_on_exit_ && !FileExists(filename)) {
+      std::cerr << "File does not exist: " << filename << std::endl;
+      abort();
+    }
+    if (!*in_) {
+      std::cerr << "Failed to open " << filename << std::endl;
+      abort();
+    }
+  }
+  ~ReadFile() {
+    if (!no_delete_on_exit_) delete in_;
+  }
+
+  inline std::istream* stream() { return in_; }
+  
+ private:
+  static bool EndsWith(const std::string& f, const std::string& suf) {
+    return (f.size() > suf.size()) && (f.rfind(suf) == f.size() - suf.size());
+  }
+  const bool no_delete_on_exit_;
+  std::istream* const in_;
+};
+
+class WriteFile {
+ public:
+  WriteFile(const std::string& filename) :
+    no_delete_on_exit_(filename == "-"),
+    out_(no_delete_on_exit_ ? static_cast<std::ostream*>(&std::cout) :
+      (EndsWith(filename, ".gz") ?
+        static_cast<std::ostream*>(new ogzstream(filename.c_str())) :
+        static_cast<std::ostream*>(new std::ofstream(filename.c_str())))) {}
+  ~WriteFile() {
+    (*out_) << std::flush;
+    if (!no_delete_on_exit_) delete out_;
+  }
+
+  inline std::ostream* stream() { return out_; }
+  
+ private:
+  static bool EndsWith(const std::string& f, const std::string& suf) {
+    return (f.size() > suf.size()) && (f.rfind(suf) == f.size() - suf.size());
+  }
+  const bool no_delete_on_exit_;
+  std::ostream* const out_;
+};
+
+#endif
diff --git a/decoder/forest_writer.cc b/decoder/forest_writer.cc
new file mode 100644
index 00000000..a9117d18
--- /dev/null
+++ b/decoder/forest_writer.cc
@@ -0,0 +1,23 @@
+#include "forest_writer.h"
+
+#include <iostream>
+
+#include <boost/lexical_cast.hpp>
+
+#include "filelib.h"
+#include "hg_io.h"
+#include "hg.h"
+
+using namespace std;
+
+ForestWriter::ForestWriter(const std::string& path, int num) :
+  fname_(path + '/' + boost::lexical_cast<string>(num) + ".json.gz"), used_(false) {}
+
+bool ForestWriter::Write(const Hypergraph& forest, bool minimal_rules) {
+  assert(!used_);
+  used_ = true;
+  cerr << "  Writing forest to " << fname_ << endl;
+  WriteFile wf(fname_);
+  return HypergraphIO::WriteToJSON(forest, minimal_rules, wf.stream());
+}
+
diff --git a/decoder/forest_writer.h b/decoder/forest_writer.h
new file mode 100644
index 00000000..819a8940
--- /dev/null
+++ b/decoder/forest_writer.h
@@ -0,0 +1,16 @@
+#ifndef _FOREST_WRITER_H_
+#define _FOREST_WRITER_H_
+
+#include <string>
+
+class Hypergraph;
+
+struct ForestWriter {
+  ForestWriter(const std::string& path, int num);
+  bool Write(const Hypergraph& forest, bool minimal_rules);
+
+  const std::string fname_;
+  bool used_;
+};
+
+#endif
diff --git a/decoder/freqdict.cc b/decoder/freqdict.cc
new file mode 100644
index 00000000..9e25d346
--- /dev/null
+++ b/decoder/freqdict.cc
@@ -0,0 +1,29 @@
+#include <iostream>
+#include <fstream>
+#include <cassert>
+#include "freqdict.h"
+#include "tdict.h"
+#include "filelib.h"
+
+using namespace std;
+
+void FreqDict::Load(const std::string& fname) {
+  cerr << "Reading word frequencies: " << fname << endl;
+  ReadFile rf(fname);
+  istream& ifs = *rf.stream();
+  int cc=0;
+  while (ifs) {
+    std::string word;
+    ifs >> word;
+    if (word.size() == 0) continue;
+    if (word[0] == '#') continue;
+    double count = 0;
+    ifs >> count;
+    assert(count > 0.0);  // use -log(f)
+    counts_[TD::Convert(word)]=count;
+    ++cc;
+    if (cc % 10000 == 0) { std::cerr << "."; }
+  }
+  std::cerr << "\n";
+  std::cerr << "Loaded " << cc << " words\n";
+}
diff --git a/decoder/freqdict.h b/decoder/freqdict.h
new file mode 100644
index 00000000..9acf0c33
--- /dev/null
+++ b/decoder/freqdict.h
@@ -0,0 +1,20 @@
+#ifndef _FREQDICT_H_
+#define _FREQDICT_H_
+
+#include <map>
+#include <string>
+#include "wordid.h"
+
+class FreqDict {
+ public:
+  void Load(const std::string& fname);
+  float LookUp(const WordID& word) const {
+    std::map<WordID,float>::const_iterator i = counts_.find(word);
+    if (i == counts_.end()) return 0;
+    return i->second;
+  }
+ private:
+  std::map<WordID, float> counts_;
+};
+
+#endif
diff --git a/decoder/fst_translator.cc b/decoder/fst_translator.cc
new file mode 100644
index 00000000..38dbd717
--- /dev/null
+++ b/decoder/fst_translator.cc
@@ -0,0 +1,91 @@
+#include "translator.h"
+
+#include <sstream>
+#include <boost/shared_ptr.hpp>
+
+#include "sentence_metadata.h"
+#include "filelib.h"
+#include "hg.h"
+#include "hg_io.h"
+#include "earley_composer.h"
+#include "phrasetable_fst.h"
+#include "tdict.h"
+
+using namespace std;
+
+struct FSTTranslatorImpl {
+  FSTTranslatorImpl(const boost::program_options::variables_map& conf) :
+      goal_sym(conf["goal"].as<string>()),
+      kGOAL_RULE(new TRule("[Goal] ||| [" + goal_sym + ",1] ||| [1]")),
+      kGOAL(TD::Convert("Goal") * -1),
+      add_pass_through_rules(conf.count("add_pass_through_rules")) {
+    fst.reset(LoadTextPhrasetable(conf["grammar"].as<vector<string> >()));
+    ec.reset(new EarleyComposer(fst.get()));
+  }
+
+  bool Translate(const string& input,
+                 const vector<double>& weights,
+                 Hypergraph* forest) {
+    bool composed = false;
+    if (input.find("{\"rules\"") == 0) {
+      istringstream is(input);
+      Hypergraph src_cfg_hg;
+      assert(HypergraphIO::ReadFromJSON(&is, &src_cfg_hg));
+      if (add_pass_through_rules) {
+        SparseVector<double> feats;
+        feats.set_value(FD::Convert("PassThrough"), 1);
+        for (int i = 0; i < src_cfg_hg.edges_.size(); ++i) {
+          const vector<WordID>& f = src_cfg_hg.edges_[i].rule_->f_;
+          for (int j = 0; j < f.size(); ++j) {
+            if (f[j] > 0) {
+              fst->AddPassThroughTranslation(f[j], feats);
+            }
+          }
+        }
+      }
+      composed = ec->Compose(src_cfg_hg, forest);
+    } else {
+      const string dummy_grammar("[" + goal_sym + "] ||| " + input + " ||| TOP=1");
+      cerr << "  Dummy grammar: " << dummy_grammar << endl;
+      istringstream is(dummy_grammar);
+      if (add_pass_through_rules) {
+        vector<WordID> words;
+        TD::ConvertSentence(input, &words);
+        SparseVector<double> feats;
+        feats.set_value(FD::Convert("PassThrough"), 1);
+        for (int i = 0; i < words.size(); ++i)
+          fst->AddPassThroughTranslation(words[i], feats);
+      }
+      composed = ec->Compose(&is, forest);
+    }
+    if (composed) {
+      Hypergraph::TailNodeVector tail(1, forest->nodes_.size() - 1);
+      Hypergraph::Node* goal = forest->AddNode(TD::Convert("Goal")*-1);
+      Hypergraph::Edge* hg_edge = forest->AddEdge(kGOAL_RULE, tail);
+      forest->ConnectEdgeToHeadNode(hg_edge, goal);
+      forest->Reweight(weights);
+    }
+    if (add_pass_through_rules)
+      fst->ClearPassThroughTranslations();
+    return composed;
+  }
+
+  const string goal_sym;
+  const TRulePtr kGOAL_RULE;
+  const WordID kGOAL;
+  const bool add_pass_through_rules;
+  boost::shared_ptr<EarleyComposer> ec;
+  boost::shared_ptr<FSTNode> fst;
+};
+
+FSTTranslator::FSTTranslator(const boost::program_options::variables_map& conf) :
+  pimpl_(new FSTTranslatorImpl(conf)) {}
+
+bool FSTTranslator::TranslateImpl(const string& input,
+                              SentenceMetadata* smeta,
+                              const vector<double>& weights,
+                              Hypergraph* minus_lm_forest) {
+  smeta->SetSourceLength(0);  // don't know how to compute this
+  return pimpl_->Translate(input, weights, minus_lm_forest);
+}
+
diff --git a/decoder/grammar.cc b/decoder/grammar.cc
new file mode 100644
index 00000000..5eb7887d
--- /dev/null
+++ b/decoder/grammar.cc
@@ -0,0 +1,148 @@
+#include "grammar.h"
+
+#include <algorithm>
+#include <utility>
+#include <map>
+
+#include "rule_lexer.h"
+#include "filelib.h"
+#include "tdict.h"
+
+using namespace std;
+
+const vector<TRulePtr> Grammar::NO_RULES;
+
+RuleBin::~RuleBin() {}
+GrammarIter::~GrammarIter() {}
+Grammar::~Grammar() {}
+
+bool Grammar::HasRuleForSpan(int i, int j, int distance) const {
+  (void) i;
+  (void) j;
+  (void) distance;
+  return true;  // always true by default
+}
+
+struct TextRuleBin : public RuleBin {
+  int GetNumRules() const {
+    return rules_.size();
+  }
+  TRulePtr GetIthRule(int i) const {
+    return rules_[i];
+  }
+  void AddRule(TRulePtr t) {
+    rules_.push_back(t);
+  }
+  int Arity() const {
+    return rules_.front()->Arity();
+  }
+  void Dump() const {
+    for (int i = 0; i < rules_.size(); ++i)
+      cerr << rules_[i]->AsString() << endl;
+  }
+ private:
+  vector<TRulePtr> rules_;
+};
+
+struct TextGrammarNode : public GrammarIter {
+  TextGrammarNode() : rb_(NULL) {}
+  ~TextGrammarNode() {
+    delete rb_;
+  }
+  const GrammarIter* Extend(int symbol) const {
+    map<WordID, TextGrammarNode>::const_iterator i = tree_.find(symbol);
+    if (i == tree_.end()) return NULL;
+    return &i->second;
+  }
+
+  const RuleBin* GetRules() const {
+    if (rb_) {
+      //rb_->Dump();
+    }
+    return rb_;
+  }
+
+  map<WordID, TextGrammarNode> tree_;
+  TextRuleBin* rb_;
+};
+
+struct TGImpl {
+  TextGrammarNode root_;
+};
+
+TextGrammar::TextGrammar() : max_span_(10), pimpl_(new TGImpl) {}
+TextGrammar::TextGrammar(const string& file) : 
+    max_span_(10),
+    pimpl_(new TGImpl) {
+  ReadFromFile(file);
+}
+
+const GrammarIter* TextGrammar::GetRoot() const {
+  return &pimpl_->root_;
+}
+
+void TextGrammar::AddRule(const TRulePtr& rule) {
+  if (rule->IsUnary()) {
+    rhs2unaries_[rule->f().front()].push_back(rule);
+    unaries_.push_back(rule);
+  } else {
+    TextGrammarNode* cur = &pimpl_->root_;
+    for (int i = 0; i < rule->f_.size(); ++i)
+      cur = &cur->tree_[rule->f_[i]];
+    if (cur->rb_ == NULL)
+      cur->rb_ = new TextRuleBin;
+    cur->rb_->AddRule(rule);
+  }
+}
+
+static void AddRuleHelper(const TRulePtr& new_rule, void* extra) {
+  static_cast<TextGrammar*>(extra)->AddRule(new_rule);
+}
+
+void TextGrammar::ReadFromFile(const string& filename) {
+  ReadFile in(filename);
+  RuleLexer::ReadRules(in.stream(), &AddRuleHelper, this);
+}
+
+bool TextGrammar::HasRuleForSpan(int i, int j, int distance) const {
+  return (max_span_ >= distance);
+}
+
+GlueGrammar::GlueGrammar(const string& file) : TextGrammar(file) {}
+
+GlueGrammar::GlueGrammar(const string& goal_nt, const string& default_nt) {
+  TRulePtr stop_glue(new TRule("[" + goal_nt + "] ||| [" + default_nt + ",1] ||| [" + default_nt + ",1]"));
+  TRulePtr glue(new TRule("[" + goal_nt + "] ||| [" + goal_nt + ",1] ["
+    + default_nt + ",2] ||| [" + goal_nt + ",1] [" + default_nt + ",2] ||| Glue=1"));
+
+  AddRule(stop_glue);
+  AddRule(glue);
+  //cerr << "GLUE: " << stop_glue->AsString() << endl;
+  //cerr << "GLUE: " << glue->AsString() << endl;
+}
+
+bool GlueGrammar::HasRuleForSpan(int i, int j, int distance) const {
+  (void) j;
+  return (i == 0);
+}
+
+PassThroughGrammar::PassThroughGrammar(const Lattice& input, const string& cat) :
+    has_rule_(input.size() + 1) {
+  for (int i = 0; i < input.size(); ++i) {
+    const vector<LatticeArc>& alts = input[i];
+    for (int k = 0; k < alts.size(); ++k) {
+      const int j = alts[k].dist2next + i;
+      has_rule_[i].insert(j);
+      const string& src = TD::Convert(alts[k].label);
+      TRulePtr pt(new TRule("[" + cat + "] ||| " + src + " ||| " + src + " ||| PassThrough=1"));
+      AddRule(pt);
+//      cerr << "PT: " << pt->AsString() << endl;
+    }
+  }
+}
+
+bool PassThroughGrammar::HasRuleForSpan(int i, int j, int distance) const {
+  const set<int>& hr = has_rule_[i];
+  if (i == j) { return !hr.empty(); }
+  return (hr.find(j) != hr.end());
+}
diff --git a/decoder/grammar.h b/decoder/grammar.h
new file mode 100644
index 00000000..46886d3a
--- /dev/null
+++ b/decoder/grammar.h
@@ -0,0 +1,89 @@
+#ifndef GRAMMAR_H_
+#define GRAMMAR_H_
+
+#include <vector>
+#include <map>
+#include <set>
+#include <boost/shared_ptr.hpp>
+#include <string>
+
+#include "lattice.h"
+#include "trule.h"
+
+struct RuleBin {
+  virtual ~RuleBin();
+  virtual int GetNumRules() const = 0;
+  virtual TRulePtr GetIthRule(int i) const = 0;
+  virtual int Arity() const = 0;
+};
+
+struct GrammarIter {
+  virtual ~GrammarIter();
+  virtual const RuleBin* GetRules() const = 0;
+  virtual const GrammarIter* Extend(int symbol) const = 0;
+};
+
+struct Grammar {
+  typedef std::map<WordID, std::vector<TRulePtr> > Cat2Rules;
+  static const std::vector<TRulePtr> NO_RULES;
+
+  virtual ~Grammar();
+  virtual const GrammarIter* GetRoot() const = 0;
+  virtual bool HasRuleForSpan(int i, int j, int distance) const;
+  const std::string GetGrammarName(){return grammar_name_;}
+  void SetGrammarName(std::string n) {grammar_name_ = n; }
+  // cat is the category to be rewritten
+  inline const std::vector<TRulePtr>& GetAllUnaryRules() const {
+    return unaries_;
+  }
+
+  // get all the unary rules that rewrite category cat
+  inline const std::vector<TRulePtr>& GetUnaryRulesForRHS(const WordID& cat) const {
+    Cat2Rules::const_iterator found = rhs2unaries_.find(cat);
+    if (found == rhs2unaries_.end())
+      return NO_RULES;
+    else
+      return found->second;
+  }
+
+ protected:
+  Cat2Rules rhs2unaries_;     // these must be filled in by subclasses!
+  std::vector<TRulePtr> unaries_;
+  std::string grammar_name_; 
+};
+
+typedef boost::shared_ptr<Grammar> GrammarPtr;
+
+class TGImpl;
+struct TextGrammar : public Grammar {
+  TextGrammar();
+  TextGrammar(const std::string& file);
+  void SetMaxSpan(int m) { max_span_ = m; }
+  
+  virtual const GrammarIter* GetRoot() const;
+  void AddRule(const TRulePtr& rule);
+  void ReadFromFile(const std::string& filename);
+  virtual bool HasRuleForSpan(int i, int j, int distance) const;
+  const std::vector<TRulePtr>& GetUnaryRules(const WordID& cat) const;
+  
+ private:
+  int max_span_;
+  boost::shared_ptr<TGImpl> pimpl_;
+  
+};
+
+struct GlueGrammar : public TextGrammar {
+  // read glue grammar from file
+  explicit GlueGrammar(const std::string& file);
+  GlueGrammar(const std::string& goal_nt, const std::string& default_nt);  // "S", "X"
+  virtual bool HasRuleForSpan(int i, int j, int distance) const;
+};
+
+struct PassThroughGrammar : public TextGrammar {
+  PassThroughGrammar(const Lattice& input, const std::string& cat);
+  virtual bool HasRuleForSpan(int i, int j, int distance) const;
+ private:
+  std::vector<std::set<int> > has_rule_;  // index by [i][j]
+};
+
+#endif
diff --git a/decoder/grammar_test.cc b/decoder/grammar_test.cc
new file mode 100644
index 00000000..62b8f958
--- /dev/null
+++ b/decoder/grammar_test.cc
@@ -0,0 +1,59 @@
+#include <cassert>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <gtest/gtest.h>
+#include "trule.h"
+#include "tdict.h"
+#include "grammar.h"
+#include "bottom_up_parser.h"
+#include "ff.h"
+#include "weights.h"
+
+using namespace std;
+
+class GrammarTest : public testing::Test {
+ public:
+  GrammarTest() {
+    wts.InitFromFile("test_data/weights.gt");
+  }
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+  Weights wts;
+};
+       
+TEST_F(GrammarTest,TestTextGrammar) {
+  vector<double> w;
+  vector<const FeatureFunction*> ms;
+  ModelSet models(w, ms);
+
+  TextGrammar g;
+  TRulePtr r1(new TRule("[X] ||| a b c ||| A B C ||| 0.1 0.2 0.3", true));
+  TRulePtr r2(new TRule("[X] ||| a b c ||| 1 2 3 ||| 0.2 0.3 0.4", true));
+  TRulePtr r3(new TRule("[X] ||| a b c d ||| A B C D ||| 0.1 0.2 0.3", true));
+  cerr << r1->AsString() << endl;
+  g.AddRule(r1);
+  g.AddRule(r2);
+  g.AddRule(r3);
+}
+
+TEST_F(GrammarTest,TestTextGrammarFile) {
+  GrammarPtr g(new TextGrammar("./test_data/grammar.prune"));
+  vector<GrammarPtr> grammars(1, g);
+
+  LatticeArc a(TD::Convert("ein"), 0.0, 1);
+  LatticeArc b(TD::Convert("haus"), 0.0, 1);
+  Lattice lattice(2);
+  lattice[0].push_back(a);
+  lattice[1].push_back(b);
+  Hypergraph forest;
+  ExhaustiveBottomUpParser parser("PHRASE", grammars);
+  parser.Parse(lattice, &forest);
+  forest.PrintGraphviz();
+}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/decoder/gzstream.cc b/decoder/gzstream.cc
new file mode 100644
index 00000000..9703e6ad
--- /dev/null
+++ b/decoder/gzstream.cc
@@ -0,0 +1,165 @@
+// ============================================================================
+// gzstream, C++ iostream classes wrapping the zlib compression library.
+// Copyright (C) 2001  Deepak Bandyopadhyay, Lutz Kettner
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+// ============================================================================
+//
+// File          : gzstream.C
+// Revision      : $Revision: 1.1 $
+// Revision_date : $Date: 2006/03/30 04:05:52 $
+// Author(s)     : Deepak Bandyopadhyay, Lutz Kettner
+// 
+// Standard streambuf implementation following Nicolai Josuttis, "The 
+// Standard C++ Library".
+// ============================================================================
+
+#include "gzstream.h"
+#include <iostream>
+#include <cstring>
+
+#ifdef GZSTREAM_NAMESPACE
+namespace GZSTREAM_NAMESPACE {
+#endif
+
+// ----------------------------------------------------------------------------
+// Internal classes to implement gzstream. See header file for user classes.
+// ----------------------------------------------------------------------------
+
+// --------------------------------------
+// class gzstreambuf:
+// --------------------------------------
+
+gzstreambuf* gzstreambuf::open( const char* name, int open_mode) {
+    if ( is_open())
+        return (gzstreambuf*)0;
+    mode = open_mode;
+    // no append nor read/write mode
+    if ((mode & std::ios::ate) || (mode & std::ios::app)
+        || ((mode & std::ios::in) && (mode & std::ios::out)))
+        return (gzstreambuf*)0;
+    char  fmode[10];
+    char* fmodeptr = fmode;
+    if ( mode & std::ios::in)
+        *fmodeptr++ = 'r';
+    else if ( mode & std::ios::out)
+        *fmodeptr++ = 'w';
+    *fmodeptr++ = 'b';
+    *fmodeptr = '\0';
+    file = gzopen( name, fmode);
+    if (file == 0)
+        return (gzstreambuf*)0;
+    opened = 1;
+    return this;
+}
+
+gzstreambuf * gzstreambuf::close() {
+    if ( is_open()) {
+        sync();
+        opened = 0;
+        if ( gzclose( file) == Z_OK)
+            return this;
+    }
+    return (gzstreambuf*)0;
+}
+
+int gzstreambuf::underflow() { // used for input buffer only
+    if ( gptr() && ( gptr() < egptr()))
+        return * reinterpret_cast<unsigned char *>( gptr());
+
+    if ( ! (mode & std::ios::in) || ! opened)
+        return EOF;
+    // Josuttis' implementation of inbuf
+    int n_putback = gptr() - eback();
+    if ( n_putback > 4)
+        n_putback = 4;
+    memcpy( buffer + (4 - n_putback), gptr() - n_putback, n_putback);
+
+    int num = gzread( file, buffer+4, bufferSize-4);
+    if (num <= 0) // ERROR or EOF
+        return EOF;
+
+    // reset buffer pointers
+    setg( buffer + (4 - n_putback),   // beginning of putback area
+          buffer + 4,                 // read position
+          buffer + 4 + num);          // end of buffer
+
+    // return next character
+    return * reinterpret_cast<unsigned char *>( gptr());    
+}
+
+int gzstreambuf::flush_buffer() {
+    // Separate the writing of the buffer from overflow() and
+    // sync() operation.
+    int w = pptr() - pbase();
+    if ( gzwrite( file, pbase(), w) != w)
+        return EOF;
+    pbump( -w);
+    return w;
+}
+
+int gzstreambuf::overflow( int c) { // used for output buffer only
+    if ( ! ( mode & std::ios::out) || ! opened)
+        return EOF;
+    if (c != EOF) {
+        *pptr() = c;
+        pbump(1);
+    }
+    if ( flush_buffer() == EOF)
+        return EOF;
+    return c;
+}
+
+int gzstreambuf::sync() {
+    // Changed to use flush_buffer() instead of overflow( EOF)
+    // which caused improper behavior with std::endl and flush(),
+    // bug reported by Vincent Ricard.
+    if ( pptr() && pptr() > pbase()) {
+        if ( flush_buffer() == EOF)
+            return -1;
+    }
+    return 0;
+}
+
+// --------------------------------------
+// class gzstreambase:
+// --------------------------------------
+
+gzstreambase::gzstreambase( const char* name, int mode) {
+    init( &buf);
+    open( name, mode);
+}
+
+gzstreambase::~gzstreambase() {
+    buf.close();
+}
+
+void gzstreambase::open( const char* name, int open_mode) {
+    if ( ! buf.open( name, open_mode))
+        clear( rdstate() | std::ios::badbit);
+}
+
+void gzstreambase::close() {
+    if ( buf.is_open())
+        if ( ! buf.close())
+            clear( rdstate() | std::ios::badbit);
+}
+
+#ifdef GZSTREAM_NAMESPACE
+} // namespace GZSTREAM_NAMESPACE
+#endif
+
+// ============================================================================
+// EOF //
diff --git a/decoder/gzstream.h b/decoder/gzstream.h
new file mode 100644
index 00000000..ad9785fd
--- /dev/null
+++ b/decoder/gzstream.h
@@ -0,0 +1,121 @@
+// ============================================================================
+// gzstream, C++ iostream classes wrapping the zlib compression library.
+// Copyright (C) 2001  Deepak Bandyopadhyay, Lutz Kettner
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+// ============================================================================
+//
+// File          : gzstream.h
+// Revision      : $Revision: 1.1 $
+// Revision_date : $Date: 2006/03/30 04:05:52 $
+// Author(s)     : Deepak Bandyopadhyay, Lutz Kettner
+// 
+// Standard streambuf implementation following Nicolai Josuttis, "The 
+// Standard C++ Library".
+// ============================================================================
+
+#ifndef GZSTREAM_H
+#define GZSTREAM_H 1
+
+// standard C++ with new header file names and std:: namespace
+#include <iostream>
+#include <fstream>
+#include <zlib.h>
+
+#ifdef GZSTREAM_NAMESPACE
+namespace GZSTREAM_NAMESPACE {
+#endif
+
+// ----------------------------------------------------------------------------
+// Internal classes to implement gzstream. See below for user classes.
+// ----------------------------------------------------------------------------
+
+class gzstreambuf : public std::streambuf {
+private:
+    static const int bufferSize = 47+256;    // size of data buff
+    // totals 512 bytes under g++ for igzstream at the end.
+
+    gzFile           file;               // file handle for compressed file
+    char             buffer[bufferSize]; // data buffer
+    char             opened;             // open/close state of stream
+    int              mode;               // I/O mode
+
+    int flush_buffer();
+public:
+    gzstreambuf() : opened(0) {
+        setp( buffer, buffer + (bufferSize-1));
+        setg( buffer + 4,     // beginning of putback area
+              buffer + 4,     // read position
+              buffer + 4);    // end position      
+        // ASSERT: both input & output capabilities will not be used together
+    }
+    int is_open() { return opened; }
+    gzstreambuf* open( const char* name, int open_mode);
+    gzstreambuf* close();
+    ~gzstreambuf() { close(); }
+    
+    virtual int     overflow( int c = EOF);
+    virtual int     underflow();
+    virtual int     sync();
+};
+
+class gzstreambase : virtual public std::ios {
+protected:
+    gzstreambuf buf;
+public:
+    gzstreambase() { init(&buf); }
+    gzstreambase( const char* name, int open_mode);
+    ~gzstreambase();
+    void open( const char* name, int open_mode);
+    void close();
+    gzstreambuf* rdbuf() { return &buf; }
+};
+
+// ----------------------------------------------------------------------------
+// User classes. Use igzstream and ogzstream analogously to ifstream and
+// ofstream respectively. They read and write files based on the gz* 
+// function interface of the zlib. Files are compatible with gzip compression.
+// ----------------------------------------------------------------------------
+
+class igzstream : public gzstreambase, public std::istream {
+public:
+    igzstream() : std::istream( &buf) {} 
+    igzstream( const char* name, int open_mode = std::ios::in)
+        : gzstreambase( name, open_mode), std::istream( &buf) {}  
+    gzstreambuf* rdbuf() { return gzstreambase::rdbuf(); }
+    void open( const char* name, int open_mode = std::ios::in) {
+        gzstreambase::open( name, open_mode);
+    }
+};
+
+class ogzstream : public gzstreambase, public std::ostream {
+public:
+    ogzstream() : std::ostream( &buf) {}
+    ogzstream( const char* name, int mode = std::ios::out)
+        : gzstreambase( name, mode), std::ostream( &buf) {}  
+    gzstreambuf* rdbuf() { return gzstreambase::rdbuf(); }
+    void open( const char* name, int open_mode = std::ios::out) {
+        gzstreambase::open( name, open_mode);
+    }
+};
+
+#ifdef GZSTREAM_NAMESPACE
+} // namespace GZSTREAM_NAMESPACE
+#endif
+
+#endif // GZSTREAM_H
+// ============================================================================
+// EOF //
+
diff --git a/decoder/hg.cc b/decoder/hg.cc
new file mode 100644
index 00000000..b56f1246
--- /dev/null
+++ b/decoder/hg.cc
@@ -0,0 +1,588 @@
+#include "hg.h"
+
+#include <algorithm>
+#include <cassert>
+#include <numeric>
+#include <set>
+#include <map>
+#include <iostream>
+
+#include "viterbi.h"
+#include "inside_outside.h"
+#include "tdict.h"
+
+using namespace std;
+
+double Hypergraph::NumberOfPaths() const {
+  return Inside<double, TransitionCountWeightFunction>(*this);
+}
+
+struct ScaledTransitionEventWeightFunction {
+  ScaledTransitionEventWeightFunction(double alpha) : scale_(alpha) {}
+  inline SparseVector<prob_t> operator()(const Hypergraph::Edge& e) const {
+    SparseVector<prob_t> result;
+    result.set_value(e.id_, e.edge_prob_.pow(scale_));
+    return result;
+  }
+  const double scale_;
+};
+
+struct TropicalValue {
+  TropicalValue() : v_() {}
+  explicit TropicalValue(int v) {
+    if (v == 0) v_ = prob_t::Zero();
+    else if (v == 1) v_ = prob_t::One();
+    else { cerr << "Bad value in TropicalValue(int).\n"; abort(); }
+  }
+  explicit TropicalValue(const prob_t& v) : v_(v) {}
+  inline TropicalValue& operator+=(const TropicalValue& o) {
+    if (v_ < o.v_) v_ = o.v_;
+    return *this;
+  }
+  inline TropicalValue& operator*=(const TropicalValue& o) {
+    v_ *= o.v_;
+    return *this;
+  }
+  inline bool operator==(const TropicalValue& o) const { return v_ == o.v_; }
+  prob_t v_;
+};
+
+struct ViterbiWeightFunction {
+  inline TropicalValue operator()(const Hypergraph::Edge& e) const {
+    return TropicalValue(e.edge_prob_);
+  }
+};
+
+struct ViterbiTransitionEventWeightFunction {
+  inline SparseVector<TropicalValue> operator()(const Hypergraph::Edge& e) const {
+    SparseVector<TropicalValue> result;
+    result.set_value(e.id_, TropicalValue(e.edge_prob_));
+    return result;
+  }
+};
+
+
+prob_t Hypergraph::ComputeEdgePosteriors(double scale, vector<prob_t>* posts) const {
+  const ScaledEdgeProb weight(scale);
+  const ScaledTransitionEventWeightFunction w2(scale);
+  SparseVector<prob_t> pv;
+  const double inside = InsideOutside<prob_t,
+                  ScaledEdgeProb,
+                  SparseVector<prob_t>,
+                  ScaledTransitionEventWeightFunction>(*this, &pv, weight, w2);
+  posts->resize(edges_.size());
+  for (int i = 0; i < edges_.size(); ++i)
+    (*posts)[i] = prob_t(pv.value(i));
+  return prob_t(inside);
+}
+
+prob_t Hypergraph::ComputeBestPathThroughEdges(vector<prob_t>* post) const {
+  SparseVector<TropicalValue> pv;
+  const TropicalValue viterbi_weight = InsideOutside<TropicalValue,
+                  ViterbiWeightFunction,
+                  SparseVector<TropicalValue>,
+                  ViterbiTransitionEventWeightFunction>(*this, &pv);
+  post->resize(edges_.size());
+  for (int i = 0; i < edges_.size(); ++i)
+    (*post)[i] = pv.value(i).v_;
+  return viterbi_weight.v_;
+}
+
+void Hypergraph::PushWeightsToSource(double scale) {
+  vector<prob_t> posts;
+  ComputeEdgePosteriors(scale, &posts);
+  for (int i = 0; i < nodes_.size(); ++i) {
+    const Hypergraph::Node& node = nodes_[i];
+    prob_t z = prob_t::Zero();
+    for (int j = 0; j < node.out_edges_.size(); ++j)
+      z += posts[node.out_edges_[j]];
+    for (int j = 0; j < node.out_edges_.size(); ++j) {
+      edges_[node.out_edges_[j]].edge_prob_ = posts[node.out_edges_[j]] / z;
+    }
+  }
+}
+
+void Hypergraph::PushWeightsToGoal(double scale) {
+  vector<prob_t> posts;
+  ComputeEdgePosteriors(scale, &posts);
+  for (int i = 0; i < nodes_.size(); ++i) {
+    const Hypergraph::Node& node = nodes_[i];
+    prob_t z = prob_t::Zero();
+    for (int j = 0; j < node.in_edges_.size(); ++j)
+      z += posts[node.in_edges_[j]];
+    for (int j = 0; j < node.in_edges_.size(); ++j) {
+      edges_[node.in_edges_[j]].edge_prob_ = posts[node.in_edges_[j]] / z;
+    }
+  }
+}
+
+struct EdgeExistsWeightFunction {
+  EdgeExistsWeightFunction(const std::vector<bool>& prunes) : prunes_(prunes) {}
+  double operator()(const Hypergraph::Edge& edge) const {
+    return (prunes_[edge.id_] ? 0.0 : 1.0);
+  }
+ private:
+  const vector<bool>& prunes_;
+};
+
+void Hypergraph::PruneEdges(const std::vector<bool>& prune_edge, bool run_inside_algorithm) {
+  assert(prune_edge.size() == edges_.size());
+  vector<bool> filtered = prune_edge;
+
+  if (run_inside_algorithm) {
+    const EdgeExistsWeightFunction wf(prune_edge);
+    // use double, not bool since vector<bool> causes problems with the Inside algorithm.
+    // I don't know a good c++ way to resolve this short of template specialization which
+    // I dislike.  If you know of a better way that doesn't involve specialization,
+    // fix this!
+    vector<double> reachable;
+    bool goal_derivable = (0 < Inside<double, EdgeExistsWeightFunction>(*this, &reachable, wf));
+    if (!goal_derivable) {
+      edges_.clear();
+      nodes_.clear();
+      nodes_.push_back(Node());
+      return;
+    }
+
+    assert(reachable.size() == nodes_.size());
+    for (int i = 0; i < edges_.size(); ++i) {
+      bool prune = prune_edge[i];
+      if (!prune) {
+        const Edge& edge = edges_[i];
+        for (int j = 0; j < edge.tail_nodes_.size(); ++j) {
+          if (!reachable[edge.tail_nodes_[j]]) {
+            prune = true;
+            break;
+          }
+        }
+      }
+      filtered[i] = prune;
+    }
+  }
+ 
+  TopologicallySortNodesAndEdges(nodes_.size() - 1, &filtered);
+}
+
+void Hypergraph::DensityPruneInsideOutside(const double scale,
+                                           const bool use_sum_prod_semiring,
+                                           const double density,
+                                           const vector<bool>* preserve_mask) {
+  assert(density >= 1.0);
+  const int plen = ViterbiPathLength(*this);
+  vector<WordID> bp;
+  int rnum = min(static_cast<int>(edges_.size()), static_cast<int>(density * static_cast<double>(plen)));
+  if (rnum == edges_.size()) {
+    cerr << "No pruning required: denisty already sufficient";
+    return;
+  }
+  vector<prob_t> io(edges_.size());
+  if (use_sum_prod_semiring)
+    ComputeEdgePosteriors(scale, &io);
+  else
+    ComputeBestPathThroughEdges(&io);
+  assert(edges_.size() == io.size());
+  vector<prob_t> sorted = io;
+  nth_element(sorted.begin(), sorted.begin() + rnum, sorted.end(), greater<prob_t>());
+  const double cutoff = sorted[rnum];
+  vector<bool> prune(edges_.size());
+  for (int i = 0; i < edges_.size(); ++i) {
+    prune[i] = (io[i] < cutoff);
+    if (preserve_mask && (*preserve_mask)[i]) prune[i] = false;
+  }
+  PruneEdges(prune);
+}
+
+void Hypergraph::BeamPruneInsideOutside(
+    const double scale,
+    const bool use_sum_prod_semiring,
+    const double alpha,
+    const vector<bool>* preserve_mask) {
+  assert(alpha > 0.0);
+  assert(scale > 0.0);
+  vector<prob_t> io(edges_.size());
+  if (use_sum_prod_semiring)
+    ComputeEdgePosteriors(scale, &io);
+  else
+    ComputeBestPathThroughEdges(&io);
+  assert(edges_.size() == io.size());
+  prob_t best;  // initializes to zero
+  for (int i = 0; i < io.size(); ++i)
+    if (io[i] > best) best = io[i];
+  const prob_t aprob(exp(-alpha));
+  const prob_t cutoff = best * aprob;
+  // cerr << "aprob = " << aprob << "\t  CUTOFF=" << cutoff << endl;
+  vector<bool> prune(edges_.size());
+  //cerr << preserve_mask.size() << " " << edges_.size() << endl;
+  int pc = 0;
+  for (int i = 0; i < io.size(); ++i) {
+    const bool prune_edge = (io[i] < cutoff);
+    if (prune_edge) ++pc;
+    prune[i] = (io[i] < cutoff);
+    if (preserve_mask && (*preserve_mask)[i]) prune[i] = false;
+  }
+  // cerr << "Beam pruning " << pc << "/" << io.size() << " edges\n";
+  PruneEdges(prune);
+}
+
+void Hypergraph::PrintGraphviz() const {
+  int ei = 0;
+  cerr << "digraph G {\n  rankdir=LR;\n  nodesep=.05;\n";
+  for (vector<Edge>::const_iterator i = edges_.begin();
+       i != edges_.end(); ++i) {
+    const Edge& edge=*i;
+    ++ei;
+    static const string none = "<null>";
+    string rule = (edge.rule_ ? edge.rule_->AsString(false) : none);
+
+    cerr << "   A_" << ei << " [label=\"" << rule << " p=" << edge.edge_prob_ 
+         << " F:" << edge.feature_values_
+         << "\" shape=\"rect\"];\n";
+    Hypergraph::TailNodeVector indorder(edge.tail_nodes_.size(), 0);
+    int ntc = 0;
+    for (int i = 0; i < edge.rule_->e_.size(); ++i) {
+      if (edge.rule_->e_[i] <= 0) indorder[ntc++] = 1 + (-1 * edge.rule_->e_[i]);
+    }
+    for (int i = 0; i < edge.tail_nodes_.size(); ++i) {
+      cerr << "     " << edge.tail_nodes_[i] << " -> A_" << ei;
+      if (edge.tail_nodes_.size() > 1) {
+        cerr << " [label=\"" << indorder[i] << "\"]";
+      }
+      cerr << ";\n";
+    }
+    cerr << "     A_" << ei << " -> " << edge.head_node_ << ";\n";
+  }
+  for (vector<Node>::const_iterator ni = nodes_.begin();
+      ni != nodes_.end(); ++ni) {
+    cerr << "  " << ni->id_ << "[label=\"" << (ni->cat_ < 0 ? TD::Convert(ni->cat_ * -1) : "")
+    //cerr << "  " << ni->id_ << "[label=\"" << ni->cat_
+         << " n=" << ni->id_
+//         << ",x=" << &*ni
+//         << ",in=" << ni->in_edges_.size() 
+//         << ",out=" << ni->out_edges_.size()
+         << "\"];\n";
+  }
+  cerr << "}\n";
+}
+
+void Hypergraph::Union(const Hypergraph& other) {
+  if (&other == this) return;
+  if (nodes_.empty()) { nodes_ = other.nodes_; edges_ = other.edges_; return; }
+  int noff = nodes_.size();
+  int eoff = edges_.size();
+  int ogoal = other.nodes_.size() - 1;
+  int cgoal = noff - 1;
+  // keep a single goal node, so add nodes.size - 1
+  nodes_.resize(nodes_.size() + ogoal);
+  // add all edges
+  edges_.resize(edges_.size() + other.edges_.size());
+
+  for (int i = 0; i < ogoal; ++i) {
+    const Node& on = other.nodes_[i];
+    Node& cn = nodes_[i + noff];
+    cn.id_ = i + noff;
+    cn.in_edges_.resize(on.in_edges_.size());
+    for (int j = 0; j < on.in_edges_.size(); ++j)
+      cn.in_edges_[j] = on.in_edges_[j] + eoff;
+
+    cn.out_edges_.resize(on.out_edges_.size());
+    for (int j = 0; j < on.out_edges_.size(); ++j)
+      cn.out_edges_[j] = on.out_edges_[j] + eoff;
+  }
+
+  for (int i = 0; i < other.edges_.size(); ++i) {
+    const Edge& oe = other.edges_[i];
+    Edge& ce = edges_[i + eoff];
+    ce.id_ = i + eoff;
+    ce.rule_ = oe.rule_;
+    ce.feature_values_ = oe.feature_values_;
+    if (oe.head_node_ == ogoal) {
+      ce.head_node_ = cgoal;
+      nodes_[cgoal].in_edges_.push_back(ce.id_);
+    } else {
+      ce.head_node_ = oe.head_node_ + noff;
+    }
+    ce.tail_nodes_.resize(oe.tail_nodes_.size());
+    for (int j = 0; j < oe.tail_nodes_.size(); ++j)
+      ce.tail_nodes_[j] = oe.tail_nodes_[j] + noff;
+  }
+
+  TopologicallySortNodesAndEdges(cgoal);
+}
+
+void Hypergraph::PruneUnreachable(int goal_node_id) {
+  TopologicallySortNodesAndEdges(goal_node_id, NULL);
+}
+
+void Hypergraph::RemoveNoncoaccessibleStates(int goal_node_id) {
+  if (goal_node_id < 0) goal_node_id += nodes_.size();
+  assert(goal_node_id >= 0);
+  assert(goal_node_id < nodes_.size());
+
+  // TODO finish implementation
+  abort();
+}
+
+struct DFSContext {
+  int node;
+  int edge_iter;
+  int tail_iter;
+  DFSContext(int n, int e, int t) : node(n), edge_iter(e), tail_iter(t) {}
+};
+
+enum ColorType { WHITE, GRAY, BLACK };
+
+template <class T>
+struct BadId {
+  bool operator()(const T& obj) const { return obj.id_ == -1; }
+};
+
+template <class T>
+struct IdCompare {
+  bool operator()(const T& a, const T& b) { return a.id_ < b.id_; }
+};
+
+void Hypergraph::TopologicallySortNodesAndEdges(int goal_index,
+                                                const vector<bool>* prune_edges) {
+  // figure out which nodes are reachable from the goal
+  vector<int> reloc_node(nodes_.size(), -1);
+  vector<int> reloc_edge(edges_.size(), -1);
+  vector<ColorType> color(nodes_.size(), WHITE);
+  vector<DFSContext> stack;
+  stack.reserve(nodes_.size());
+  stack.push_back(DFSContext(goal_index, 0, 0));
+  int node_count = 0;
+  int edge_count = 0;
+  while(!stack.empty()) {
+    const DFSContext& p = stack.back();
+    int cur_ni = p.node;
+    int edge_i = p.edge_iter;
+    int tail_i = p.tail_iter;
+    stack.pop_back();
+    const Node* cur_node = &nodes_[cur_ni];
+    int edge_end = cur_node->in_edges_.size();
+    while (edge_i != edge_end) {
+      const Edge& cur_edge = edges_[cur_node->in_edges_[edge_i]];
+      const int tail_end = cur_edge.tail_nodes_.size();
+      if ((tail_end == tail_i) || (prune_edges && (*prune_edges)[cur_edge.id_])) {
+        ++edge_i;
+        tail_i = 0;
+        continue;
+      }
+      const int tail_ni = cur_edge.tail_nodes_[tail_i];
+      const int tail_color = color[tail_ni];
+      if (tail_color == WHITE) {
+        stack.push_back(DFSContext(cur_ni, edge_i, ++tail_i));
+        cur_ni = tail_ni;
+        cur_node = &nodes_[cur_ni];
+        color[cur_ni] = GRAY;
+        edge_i = 0;
+        edge_end = cur_node->in_edges_.size();
+        tail_i = 0;
+      } else if (tail_color == BLACK) {
+        ++tail_i;
+      } else if (tail_color == GRAY) {
+        // this can happen if, e.g., it is possible to rederive
+        // a single cell in the CKY chart via a cycle.
+        cerr << "Detected forbidden cycle in HG:\n";
+        cerr << "  " << cur_edge.rule_->AsString() << endl;
+        while(!stack.empty()) {
+          const DFSContext& p = stack.back();
+          cerr << "  " << edges_[nodes_[p.node].in_edges_[p.edge_iter]].rule_->AsString() << endl;
+          stack.pop_back();
+        }
+        abort();
+      }
+    }
+    color[cur_ni] = BLACK;
+    reloc_node[cur_ni] = node_count++;
+    if (prune_edges) {
+      for (int i = 0; i < edge_end; ++i) {
+        int ei = cur_node->in_edges_[i];
+        if (!(*prune_edges)[ei])
+          reloc_edge[cur_node->in_edges_[i]] = edge_count++;
+      }
+    } else {
+      for (int i = 0; i < edge_end; ++i)
+        reloc_edge[cur_node->in_edges_[i]] = edge_count++;
+    }
+  }
+#ifndef HG_EDGES_TOPO_SORTED
+  int ec = 0;
+  for (int i = 0; i < reloc_edge.size(); ++i) {
+    int& cp = reloc_edge[i];
+    if (cp >= 0) { cp = ec++; }
+  }
+#endif
+
+#if 0
+  cerr << "TOPO:";
+  for (int i = 0; i < reloc_node.size(); ++i)
+    cerr << " " << reloc_node[i];
+  cerr << endl;
+  cerr << "EDGE:";
+  for (int i = 0; i < reloc_edge.size(); ++i)
+    cerr << " " << reloc_edge[i];
+  cerr << endl;
+#endif
+  bool no_op = true;
+  for (int i = 0; i < reloc_node.size() && no_op; ++i)
+    if (reloc_node[i] != i) no_op = false;
+  for (int i = 0; i < reloc_edge.size() && no_op; ++i)
+    if (reloc_edge[i] != i) no_op = false;
+  if (no_op) return;
+  for (int i = 0; i < reloc_node.size(); ++i) {
+    Node& node = nodes_[i];
+    node.id_ = reloc_node[i];
+    int c = 0;
+    for (int j = 0; j < node.in_edges_.size(); ++j) {
+      const int new_index = reloc_edge[node.in_edges_[j]];
+      if (new_index >= 0)
+        node.in_edges_[c++] = new_index;
+    }
+    node.in_edges_.resize(c);
+    c = 0;
+    for (int j = 0; j < node.out_edges_.size(); ++j) {
+      const int new_index = reloc_edge[node.out_edges_[j]];
+      if (new_index >= 0)
+        node.out_edges_[c++] = new_index;
+    }
+    node.out_edges_.resize(c);
+  }
+  for (int i = 0; i < reloc_edge.size(); ++i) {
+    Edge& edge = edges_[i];
+    edge.id_ = reloc_edge[i];
+    edge.head_node_ = reloc_node[edge.head_node_];
+    for (int j = 0; j < edge.tail_nodes_.size(); ++j)
+      edge.tail_nodes_[j] = reloc_node[edge.tail_nodes_[j]];
+  }
+  edges_.erase(remove_if(edges_.begin(), edges_.end(), BadId<Edge>()), edges_.end());
+  nodes_.erase(remove_if(nodes_.begin(), nodes_.end(), BadId<Node>()), nodes_.end());
+  sort(nodes_.begin(), nodes_.end(), IdCompare<Node>());
+#ifndef HG_EDGES_TOPO_SORTED
+  sort(edges_.begin(), edges_.end(), IdCompare<Edge>());
+#endif
+}
+
+TRulePtr Hypergraph::kEPSRule;
+TRulePtr Hypergraph::kUnaryRule;
+
+void Hypergraph::EpsilonRemove(WordID eps) {
+  if (!kEPSRule) {
+    kEPSRule.reset(new TRule("[X] ||| <eps> ||| <eps>"));
+    kUnaryRule.reset(new TRule("[X] ||| [X,1] ||| [X,1]"));
+  }
+  vector<bool> kill(edges_.size(), false);
+  for (int i = 0; i < edges_.size(); ++i) {
+    const Edge& edge = edges_[i];
+    if (edge.tail_nodes_.empty() &&
+        edge.rule_->f_.size() == 1 &&
+        edge.rule_->f_[0] == eps) {
+      kill[i] = true;
+      if (!edge.feature_values_.empty()) {
+        Node& node = nodes_[edge.head_node_];
+        if (node.in_edges_.size() != 1) {
+          cerr << "[WARNING] <eps> edge with features going into non-empty node - can't promote\n";
+          // this *probably* means that there are multiple derivations of the
+          // same sequence via different paths through the input forest
+          // this needs to be investigated and fixed
+        } else {
+          for (int j = 0; j < node.out_edges_.size(); ++j)
+            edges_[node.out_edges_[j]].feature_values_ += edge.feature_values_;
+          // cerr << "PROMOTED " << edge.feature_values_ << endl;
+	}
+      }
+    }
+  }
+  bool created_eps = false;
+  PruneEdges(kill);
+  for (int i = 0; i < nodes_.size(); ++i) {
+    const Node& node = nodes_[i];
+    if (node.in_edges_.empty()) {
+      for (int j = 0; j < node.out_edges_.size(); ++j) {
+        Edge& edge = edges_[node.out_edges_[j]];
+        if (edge.rule_->Arity() == 2) {
+          assert(edge.rule_->f_.size() == 2);
+          assert(edge.rule_->e_.size() == 2);
+          edge.rule_ = kUnaryRule;
+          int cur = node.id_;
+          int t = -1;
+          assert(edge.tail_nodes_.size() == 2);
+          for (int i = 0; i < 2; ++i) if (edge.tail_nodes_[i] != cur) { t = edge.tail_nodes_[i]; }
+          assert(t != -1);
+          edge.tail_nodes_.resize(1);
+          edge.tail_nodes_[0] = t;
+        } else {
+          edge.rule_ = kEPSRule;
+          edge.rule_->f_[0] = eps;
+          edge.rule_->e_[0] = eps;
+          edge.tail_nodes_.clear();
+          created_eps = true;
+        }
+      }
+    }
+  }
+  vector<bool> k2(edges_.size(), false);
+  PruneEdges(k2);
+  if (created_eps) EpsilonRemove(eps);
+}
+
+struct EdgeWeightSorter {
+  const Hypergraph& hg;
+  EdgeWeightSorter(const Hypergraph& h) : hg(h) {}
+  bool operator()(int a, int b) const {
+    return hg.edges_[a].edge_prob_ > hg.edges_[b].edge_prob_;
+  }
+};
+
+void Hypergraph::SortInEdgesByEdgeWeights() {
+  for (int i = 0; i < nodes_.size(); ++i) {
+    Node& node = nodes_[i];
+    sort(node.in_edges_.begin(), node.in_edges_.end(), EdgeWeightSorter(*this));
+  }
+}
+
+Hypergraph* Hypergraph::CreateViterbiHypergraph(const vector<bool>* edges) const {
+  vector<const Edge*> vit_edges;
+  if (edges) {
+    assert(edges->size() == edges_.size());
+    Viterbi<vector<const Edge*>, ViterbiPathTraversal, prob_t, EdgeSelectEdgeWeightFunction>(*this, &vit_edges, ViterbiPathTraversal(), EdgeSelectEdgeWeightFunction(*edges));
+  } else {
+    Viterbi<vector<const Edge*>, ViterbiPathTraversal, prob_t, EdgeProb>(*this, &vit_edges);
+  }
+  map<int, int> old2new_node;
+  int num_new_nodes = 0;
+  for (int i = 0; i < vit_edges.size(); ++i) {
+    const Edge& edge = *vit_edges[i];
+    for (int j = 0; j < edge.tail_nodes_.size(); ++j)
+      assert(old2new_node.count(edge.tail_nodes_[j]) > 0);
+    if (old2new_node.count(edge.head_node_) == 0) {
+      old2new_node[edge.head_node_] = num_new_nodes;
+      ++num_new_nodes;
+    }
+  }
+  Hypergraph* out = new Hypergraph(num_new_nodes, vit_edges.size(), is_linear_chain_);
+  for (map<int, int>::iterator it = old2new_node.begin();
+       it != old2new_node.end(); ++it) {
+    const Node& old_node = nodes_[it->first];
+    Node& new_node = out->nodes_[it->second];
+    new_node.cat_ = old_node.cat_;
+    new_node.id_ = it->second;
+  }
+
+  for (int i = 0; i < vit_edges.size(); ++i) {
+    const Edge& old_edge = *vit_edges[i];
+    Edge& new_edge = out->edges_[i];
+    new_edge = old_edge;
+    new_edge.id_ = i;
+    const int new_head_node = old2new_node[old_edge.head_node_];
+    new_edge.head_node_ = new_head_node;
+    out->nodes_[new_head_node].in_edges_.push_back(i);
+    for (int j = 0; j < old_edge.tail_nodes_.size(); ++j) {
+      const int new_tail_node = old2new_node[old_edge.tail_nodes_[j]];
+      new_edge.tail_nodes_[j] = new_tail_node;
+      out->nodes_[new_tail_node].out_edges_.push_back(i);
+    }
+  }
+  return out;
+}
+
diff --git a/decoder/hg.h b/decoder/hg.h
new file mode 100644
index 00000000..8d056358
--- /dev/null
+++ b/decoder/hg.h
@@ -0,0 +1,247 @@
+#ifndef _HG_H_
+#define _HG_H_
+
+#include <string>
+#include <vector>
+
+#include "small_vector.h"
+#include "sparse_vector.h"
+#include "wordid.h"
+#include "trule.h"
+#include "prob.h"
+
+// if you define this, edges_ will be sorted
+// (normally, just nodes_ are), but this can be quite
+// slow
+#undef HG_EDGES_TOPO_SORTED
+
+// class representing an acyclic hypergraph
+//  - edges have 1 head, 0..n tails
+class Hypergraph {
+ public:
+  Hypergraph() : is_linear_chain_(false) {}
+
+  // SmallVector is a fast, small vector<int> implementation for sizes <= 2
+  typedef SmallVector TailNodeVector;
+
+  // TODO get rid of cat_?
+  struct Node {
+    Node() : id_(), cat_() {}
+    int id_; // equal to this object's position in the nodes_ vector
+    WordID cat_;  // non-terminal category if <0, 0 if not set
+    std::vector<int> in_edges_;   // contents refer to positions in edges_
+    std::vector<int> out_edges_;  // contents refer to positions in edges_
+  };
+
+  // TODO get rid of edge_prob_? (can be computed on the fly as the dot
+  // product of the weight vector and the feature values)
+  struct Edge {
+    Edge() : i_(-1), j_(-1), prev_i_(-1), prev_j_(-1) {}
+    inline int Arity() const { return tail_nodes_.size(); }
+    int head_node_;               // refers to a position in nodes_
+    TailNodeVector tail_nodes_;   // contents refer to positions in nodes_
+    TRulePtr rule_;
+    SparseVector<double> feature_values_;
+    prob_t edge_prob_;             // dot product of weights and feat_values
+    int id_;   // equal to this object's position in the edges_ vector
+
+    // span info. typically, i_ and j_ refer to indices in the source sentence
+    // if a synchronous parse has been executed i_ and j_ will refer to indices
+    // in the target sentence / lattice and prev_i_ prev_j_ will refer to
+    // positions in the source.  Note: it is up to the translator implementation
+    // to properly set these values.  For some models (like the Forest-input
+    // phrase based model) it may not be straightforward to do.  if these values
+    // are not properly set, most things will work but alignment and any features
+    // that depend on them will be broken.
+    short int i_;
+    short int j_;
+    short int prev_i_;
+    short int prev_j_;
+  };
+
+  void swap(Hypergraph& other) {
+    other.nodes_.swap(nodes_);
+    std::swap(is_linear_chain_, other.is_linear_chain_);
+    other.edges_.swap(edges_);
+  }
+
+  void ResizeNodes(int size) {
+    nodes_.resize(size);
+    for (int i = 0; i < size; ++i) nodes_[i].id_ = i;
+  }
+
+  // reserves space in the nodes vector to prevent memory locations
+  // from changing
+  void ReserveNodes(size_t n, size_t e = 0) {
+    nodes_.reserve(n);
+    if (e) edges_.reserve(e);
+  }
+
+  Edge* AddEdge(const TRulePtr& rule, const TailNodeVector& tail) {
+    edges_.push_back(Edge());
+    Edge* edge = &edges_.back();
+    edge->rule_ = rule;
+    edge->tail_nodes_ = tail;
+    edge->id_ = edges_.size() - 1;
+    for (int i = 0; i < edge->tail_nodes_.size(); ++i)
+      nodes_[edge->tail_nodes_[i]].out_edges_.push_back(edge->id_);
+    return edge;
+  }
+
+  Node* AddNode(const WordID& cat) {
+    nodes_.push_back(Node());
+    nodes_.back().cat_ = cat;
+    nodes_.back().id_ = nodes_.size() - 1;
+    return &nodes_.back();
+  }
+
+  void ConnectEdgeToHeadNode(const int edge_id, const int head_id) {
+    edges_[edge_id].head_node_ = head_id;
+    nodes_[head_id].in_edges_.push_back(edge_id);
+  }
+
+  // TODO remove this - use the version that takes indices
+  void ConnectEdgeToHeadNode(Edge* edge, Node* head) {
+    edge->head_node_ = head->id_;
+    head->in_edges_.push_back(edge->id_);
+  }
+
+  // merge the goal node from other with this goal node
+  void Union(const Hypergraph& other);
+
+  void PrintGraphviz() const;
+
+  // compute the total number of paths in the forest
+  double NumberOfPaths() const;
+
+  // BEWARE. this assumes that the source and target language
+  // strings are identical and that there are no loops.
+  // It assumes a bunch of other things about where the
+  // epsilons will be.  It tries to assert failure if you
+  // break these assumptions, but it may not.
+  // TODO - make this work
+  void EpsilonRemove(WordID eps);
+
+  // multiple the weights vector by the edge feature vector
+  // (inner product) to set the edge probabilities
+  template <typename V>
+  void Reweight(const V& weights) {
+    for (int i = 0; i < edges_.size(); ++i) {
+      Edge& e = edges_[i];
+      e.edge_prob_.logeq(e.feature_values_.dot(weights));
+    }
+  }
+
+  // computes inside and outside scores for each
+  // edge in the hypergraph
+  // alpha->size = edges_.size = beta->size
+  // returns inside prob of goal node
+  prob_t ComputeEdgePosteriors(double scale,
+                               std::vector<prob_t>* posts) const;
+
+  // find the score of the very best path passing through each edge
+  prob_t ComputeBestPathThroughEdges(std::vector<prob_t>* posts) const;
+
+  // create a new hypergraph consisting only of the nodes / edges
+  // in the Viterbi derivation of this hypergraph
+  // if edges is set, use the EdgeSelectEdgeWeightFunction
+  Hypergraph* CreateViterbiHypergraph(const std::vector<bool>* edges = NULL) const;
+
+  // move weights as near to the source as possible, resulting in a
+  // stochastic automaton.  ONLY FUNCTIONAL FOR *LATTICES*.
+  // See M. Mohri and M. Riley. A Weight Pushing Algorithm for Large
+  //   Vocabulary Speech Recognition. 2001.
+  // the log semiring (NOT tropical) is used
+  void PushWeightsToSource(double scale = 1.0);
+  // same, except weights are pushed to the goal, works for HGs,
+  // not just lattices
+  void PushWeightsToGoal(double scale = 1.0);
+
+  void SortInEdgesByEdgeWeights();
+
+  void PruneUnreachable(int goal_node_id); // DEPRECATED
+
+  void RemoveNoncoaccessibleStates(int goal_node_id = -1);
+
+  // remove edges from the hypergraph if prune_edge[edge_id] is true
+  // TODO need to investigate why this shouldn't be run for the forest trans
+  // case.  To investigate, change false to true and see where ftrans crashes
+  void PruneEdges(const std::vector<bool>& prune_edge, bool run_inside_algorithm = false);
+
+  // if you don't know, use_sum_prod_semiring should be false
+  void DensityPruneInsideOutside(const double scale, const bool use_sum_prod_semiring, const double density,
+                                 const std::vector<bool>* preserve_mask = NULL);
+
+  // prunes any edge whose score on the best path taking that edge is more than alpha away
+  // from the score of the global best past (or the highest edge posterior)
+  void BeamPruneInsideOutside(const double scale, const bool use_sum_prod_semiring, const double alpha,
+                              const std::vector<bool>* preserve_mask = NULL);
+
+  void clear() {
+    nodes_.clear();
+    edges_.clear();
+  }
+
+  inline size_t NumberOfEdges() const { return edges_.size(); }
+  inline size_t NumberOfNodes() const { return nodes_.size(); }
+  inline bool empty() const { return nodes_.empty(); }
+
+  // linear chains can be represented in a number of ways in a hypergraph,
+  // we define them to consist only of lexical translations and monotonic rules
+  inline bool IsLinearChain() const { return is_linear_chain_; }
+  bool is_linear_chain_;
+
+  // nodes_ is sorted in topological order
+  std::vector<Node> nodes_;
+  // edges_ is not guaranteed to be in any particular order
+  std::vector<Edge> edges_;
+
+  // reorder nodes_ so they are in topological order
+  // source nodes at 0 sink nodes at size-1
+  void TopologicallySortNodesAndEdges(int goal_idx,
+                                      const std::vector<bool>* prune_edges = NULL);
+ private:
+  Hypergraph(int num_nodes, int num_edges, bool is_lc) : is_linear_chain_(is_lc), nodes_(num_nodes), edges_(num_edges) {}
+
+  static TRulePtr kEPSRule;
+  static TRulePtr kUnaryRule;
+};
+
+// common WeightFunctions, map an edge -> WeightType
+// for generic Viterbi/Inside algorithms
+struct EdgeProb {
+  inline const prob_t& operator()(const Hypergraph::Edge& e) const { return e.edge_prob_; }
+};
+
+struct EdgeSelectEdgeWeightFunction {
+  EdgeSelectEdgeWeightFunction(const std::vector<bool>& v) : v_(v) {}
+  inline prob_t operator()(const Hypergraph::Edge& e) const {
+    if (v_[e.id_]) return prob_t::One();
+    else return prob_t::Zero();
+  }
+ private:
+  const std::vector<bool>& v_;
+};
+
+struct ScaledEdgeProb {
+  ScaledEdgeProb(const double& alpha) : alpha_(alpha) {}
+  inline prob_t operator()(const Hypergraph::Edge& e) const { return e.edge_prob_.pow(alpha_); }
+  const double alpha_;
+};
+
+// see Li (2010), Section 3.2.2-- this is 'x_e = p_e*r_e'
+struct EdgeFeaturesAndProbWeightFunction {
+  inline const SparseVector<prob_t> operator()(const Hypergraph::Edge& e) const {
+    SparseVector<prob_t> res;
+    for (SparseVector<double>::const_iterator it = e.feature_values_.begin();
+         it != e.feature_values_.end(); ++it)
+      res.set_value(it->first, prob_t(it->second) * e.edge_prob_);
+    return res;
+  }
+};
+
+struct TransitionCountWeightFunction {
+  inline double operator()(const Hypergraph::Edge& e) const { (void)e; return 1.0; }
+};
+
+#endif
diff --git a/decoder/hg_intersect.cc b/decoder/hg_intersect.cc
new file mode 100644
index 00000000..02ff752e
--- /dev/null
+++ b/decoder/hg_intersect.cc
@@ -0,0 +1,160 @@
+#include "hg_intersect.h"
+
+#include <vector>
+#include <tr1/unordered_map>
+#include <boost/lexical_cast.hpp>
+#include <boost/functional/hash.hpp>
+
+#include "tdict.h"
+#include "hg.h"
+#include "trule.h"
+#include "wordid.h"
+#include "bottom_up_parser.h"
+
+using boost::lexical_cast;
+using namespace std::tr1;
+using namespace std;
+
+struct RuleFilter {
+  unordered_map<vector<WordID>, bool, boost::hash<vector<WordID> > > exists_;
+  bool true_lattice;
+  RuleFilter(const Lattice& target, int max_phrase_size) {
+    true_lattice = false;
+    for (int i = 0; i < target.size(); ++i) {
+      vector<WordID> phrase;
+      int lim = min(static_cast<int>(target.size()), i + max_phrase_size);
+      for (int j = i; j < lim; ++j) {
+        if (target[j].size() > 1) { true_lattice = true; break; }
+        phrase.push_back(target[j][0].label);
+        exists_[phrase] = true;
+      }
+    }
+    vector<WordID> sos(1, TD::Convert("<s>"));
+    exists_[sos] = true;
+  }
+  bool operator()(const TRule& r) const {
+    // TODO do some smarter filtering for lattices
+    if (true_lattice) return false;  // don't filter "true lattice" input
+    const vector<WordID>& e = r.e();
+    for (int i = 0; i < e.size(); ++i) {
+      if (e[i] <= 0) continue;
+      vector<WordID> phrase;
+      for (int j = i; j < e.size(); ++j) {
+        if (e[j] <= 0) break;
+        phrase.push_back(e[j]);
+        if (exists_.count(phrase) == 0) return true;
+      }
+    }
+    return false;
+  }
+};
+
+static bool FastLinearIntersect(const Lattice& target, Hypergraph* hg) {
+  cerr << "  Fast linear-chain intersection...\n";
+  vector<bool> prune(hg->edges_.size(), false);
+  set<int> cov;
+  map<const TRule*, TRulePtr> inverted_rules;
+  for (int i = 0; i < prune.size(); ++i) {
+    Hypergraph::Edge& edge = hg->edges_[i];
+    if (edge.Arity() == 0) {
+      const int trg_index = edge.prev_i_;
+      const WordID trg = target[trg_index][0].label;
+      assert(edge.rule_->EWords() == 1);
+      TRulePtr& inv_rule = inverted_rules[edge.rule_.get()];
+      if (!inv_rule) {
+        inv_rule.reset(new TRule(*edge.rule_));
+        inv_rule->e_.swap(inv_rule->f_);
+      }
+      prune[i] = (edge.rule_->e_[0] != trg);
+      if (!prune[i]) {
+        cov.insert(trg_index);
+        swap(edge.prev_i_, edge.i_);
+        swap(edge.prev_j_, edge.j_);
+        edge.rule_.swap(inv_rule);
+      }
+    }
+  }
+  hg->PruneEdges(prune, true);
+  return (cov.size() == target.size());
+}
+
+bool HG::Intersect(const Lattice& target, Hypergraph* hg) {
+  // there are a number of faster algorithms available for restricted
+  // classes of hypergraph and/or target.
+  if (hg->IsLinearChain() && target.IsSentence())
+    return FastLinearIntersect(target, hg);
+
+  vector<bool> rem(hg->edges_.size(), false);
+  const RuleFilter filter(target, 15);   // TODO make configurable
+  for (int i = 0; i < rem.size(); ++i)
+    rem[i] = filter(*hg->edges_[i].rule_);
+  hg->PruneEdges(rem, true);
+
+  const int nedges = hg->edges_.size();
+  const int nnodes = hg->nodes_.size();
+
+  TextGrammar* g = new TextGrammar;
+  GrammarPtr gp(g);
+  vector<int> cats(nnodes);
+  // each node in the translation forest becomes a "non-terminal" in the new
+  // grammar, create the labels here
+  const string kSEP = "_";
+  for (int i = 0; i < nnodes; ++i) {
+    const char* pstr = "CAT";
+    if (hg->nodes_[i].cat_ < 0)
+      pstr = TD::Convert(-hg->nodes_[i].cat_);
+    cats[i] = TD::Convert(pstr + kSEP + lexical_cast<string>(i)) * -1;
+  }
+
+  // construct the grammar
+  for (int i = 0; i < nedges; ++i) {
+    const Hypergraph::Edge& edge = hg->edges_[i];
+    const vector<WordID>& tgt = edge.rule_->e();
+    const vector<WordID>& src = edge.rule_->f();
+    TRulePtr rule(new TRule);
+    rule->prev_i = edge.i_;
+    rule->prev_j = edge.j_;
+    rule->lhs_ = cats[edge.head_node_];
+    vector<WordID>& f = rule->f_;
+    vector<WordID>& e = rule->e_;
+    f.resize(tgt.size());   // swap source and target, since the parser
+    e.resize(src.size());   // parses using the source side!
+    Hypergraph::TailNodeVector tn(edge.tail_nodes_.size());
+    int ntc = 0;
+    for (int j = 0; j < tgt.size(); ++j) {
+      const WordID& cur = tgt[j];
+      if (cur > 0) {
+        f[j] = cur;
+      } else {
+        tn[ntc++] = cur;
+        f[j] = cats[edge.tail_nodes_[-cur]];
+      }
+    }
+    ntc = 0;
+    for (int j = 0; j < src.size(); ++j) {
+      const WordID& cur = src[j];
+      if (cur > 0) {
+        e[j] = cur;
+      } else {
+        e[j] = tn[ntc++];
+      }
+    }
+    rule->scores_ = edge.feature_values_;
+    rule->parent_rule_ = edge.rule_;
+    rule->ComputeArity();
+    //cerr << "ADD: " << rule->AsString() << endl;
+    
+    g->AddRule(rule);
+  }
+  g->SetMaxSpan(target.size() + 1);
+  const string& new_goal = TD::Convert(cats.back() * -1);
+  vector<GrammarPtr> grammars(1, gp);
+  Hypergraph tforest;
+  ExhaustiveBottomUpParser parser(new_goal, grammars);
+  if (!parser.Parse(target, &tforest))
+    return false;
+  else
+    hg->swap(tforest);
+  return true;
+}
+
diff --git a/decoder/hg_intersect.h b/decoder/hg_intersect.h
new file mode 100644
index 00000000..826bdaae
--- /dev/null
+++ b/decoder/hg_intersect.h
@@ -0,0 +1,13 @@
+#ifndef _HG_INTERSECT_H_
+#define _HG_INTERSECT_H_
+
+#include <vector>
+
+#include "lattice.h"
+
+class Hypergraph;
+struct HG {
+  static bool Intersect(const Lattice& target, Hypergraph* hg);
+};
+
+#endif
diff --git a/decoder/hg_io.cc b/decoder/hg_io.cc
new file mode 100644
index 00000000..5161931d
--- /dev/null
+++ b/decoder/hg_io.cc
@@ -0,0 +1,673 @@
+#include "hg_io.h"
+
+#include <sstream>
+#include <iostream>
+
+#include <boost/lexical_cast.hpp>
+
+#include "tdict.h"
+#include "json_parse.h"
+#include "hg.h"
+
+using namespace std;
+
+struct HGReader : public JSONParser {
+  HGReader(Hypergraph* g) : rp("[X] ||| "), state(-1), hg(*g), nodes_needed(true), edges_needed(true) { nodes = 0; edges = 0; }
+
+  void CreateNode(const string& cat, const vector<int>& in_edges) {
+    WordID c = TD::Convert("X") * -1;
+    if (!cat.empty()) c = TD::Convert(cat) * -1;
+    Hypergraph::Node* node = hg.AddNode(c);
+    for (int i = 0; i < in_edges.size(); ++i) {
+      if (in_edges[i] >= hg.edges_.size()) {
+        cerr << "JSONParser: in_edges[" << i << "]=" << in_edges[i]
+             << ", but hg only has " << hg.edges_.size() << " edges!\n";
+        abort();
+      }
+      hg.ConnectEdgeToHeadNode(&hg.edges_[in_edges[i]], node);
+    }
+  }
+  void CreateEdge(const TRulePtr& rule, SparseVector<double>* feats, const SmallVector& tail) {
+    Hypergraph::Edge* edge = hg.AddEdge(rule, tail);
+    feats->swap(edge->feature_values_);
+    edge->i_ = spans[0];
+    edge->j_ = spans[1];
+    edge->prev_i_ = spans[2];
+    edge->prev_j_ = spans[3];
+  }
+
+  bool HandleJSONEvent(int type, const JSON_value* value) {
+    switch(state) {
+    case -1:
+      assert(type == JSON_T_OBJECT_BEGIN);
+      state = 0;
+      break;
+    case 0:
+      if (type == JSON_T_OBJECT_END) {
+        //cerr << "HG created\n";  // TODO, signal some kind of callback
+      } else if (type == JSON_T_KEY) {
+        string val = value->vu.str.value;
+        if (val == "features") { assert(fdict.empty()); state = 1; }
+        else if (val == "is_sorted") { state = 3; }
+        else if (val == "rules") { assert(rules.empty()); state = 4; }
+        else if (val == "node") { state = 8; }
+        else if (val == "edges") { state = 13; }
+        else { cerr << "Unexpected key: " << val << endl; return false; }
+      }
+      break;
+
+    // features
+    case 1:
+      if(type == JSON_T_NULL) { state = 0; break; }
+      assert(type == JSON_T_ARRAY_BEGIN);
+      state = 2;
+      break;
+    case 2:
+      if(type == JSON_T_ARRAY_END) { state = 0; break; }
+      assert(type == JSON_T_STRING);
+      fdict.push_back(FD::Convert(value->vu.str.value));
+      assert(fdict.back() > 0);
+      break;
+
+    // is_sorted
+    case 3:
+      assert(type == JSON_T_TRUE || type == JSON_T_FALSE);
+      is_sorted = (type == JSON_T_TRUE);
+      if (!is_sorted) { cerr << "[WARNING] is_sorted flag is ignored\n"; }
+      state = 0;
+      break;
+
+    // rules
+    case 4:
+      if(type == JSON_T_NULL) { state = 0; break; }
+      assert(type == JSON_T_ARRAY_BEGIN);
+      state = 5;
+      break;
+    case 5:
+      if(type == JSON_T_ARRAY_END) { state = 0; break; }
+      assert(type == JSON_T_INTEGER);
+      state = 6;
+      rule_id = value->vu.integer_value;
+      break;
+    case 6:
+      assert(type == JSON_T_STRING);
+      rules[rule_id] = TRulePtr(new TRule(value->vu.str.value));
+      state = 5;
+      break;
+
+    // Nodes
+    case 8:
+      assert(type == JSON_T_OBJECT_BEGIN);
+      ++nodes;
+      in_edges.clear();
+      cat.clear();
+      state = 9; break;
+    case 9:
+      if (type == JSON_T_OBJECT_END) {
+        //cerr << "Creating NODE\n";
+        CreateNode(cat, in_edges);
+        state = 0; break;
+      }
+      assert(type == JSON_T_KEY);
+      cur_key = value->vu.str.value;
+      if (cur_key == "cat") { assert(cat.empty()); state = 10; break; }
+      if (cur_key == "in_edges") { assert(in_edges.empty()); state = 11; break; }
+      cerr << "Syntax error: unexpected key " << cur_key << " in node specification.\n";
+      return false;
+    case 10:
+      assert(type == JSON_T_STRING || type == JSON_T_NULL);
+      cat = value->vu.str.value;
+      state = 9; break;
+    case 11:
+      if (type == JSON_T_NULL) { state = 9; break; }
+      assert(type == JSON_T_ARRAY_BEGIN);
+      state = 12; break;
+    case 12:
+      if (type == JSON_T_ARRAY_END) { state = 9; break; }
+      assert(type == JSON_T_INTEGER);
+      //cerr << "in_edges: " << value->vu.integer_value << endl;
+      in_edges.push_back(value->vu.integer_value);
+      break;
+
+    //   "edges": [ { "tail": null, "feats" : [0,1.63,1,-0.54], "rule": 12},
+    //         { "tail": null, "feats" : [0,0.87,1,0.02], "spans":[1,2,3,4], "rule": 17},
+    //         { "tail": [0], "feats" : [1,2.3,2,15.3,"ExtraFeature",1.2], "rule": 13}]
+    case 13:
+      assert(type == JSON_T_ARRAY_BEGIN);
+      state = 14;
+      break;
+    case 14:
+      if (type == JSON_T_ARRAY_END) { state = 0; break; }
+      assert(type == JSON_T_OBJECT_BEGIN);
+      //cerr << "New edge\n";
+      ++edges;
+      cur_rule.reset(); feats.clear(); tail.clear();
+      state = 15; break;
+    case 15:
+      if (type == JSON_T_OBJECT_END) {
+        CreateEdge(cur_rule, &feats, tail);
+        state = 14; break;
+      }
+      assert(type == JSON_T_KEY);
+      cur_key = value->vu.str.value;
+      //cerr << "edge key " << cur_key << endl;
+      if (cur_key == "rule") { assert(!cur_rule); state = 16; break; }
+      if (cur_key == "spans") { assert(!cur_rule); state = 22; break; }
+      if (cur_key == "feats") { assert(feats.empty()); state = 17; break; }
+      if (cur_key == "tail") { assert(tail.empty()); state = 20; break; }
+      cerr << "Unexpected key " << cur_key << " in edge specification\n";
+      return false;
+    case 16:    // edge.rule
+      if (type == JSON_T_INTEGER) {
+        int rule_id = value->vu.integer_value;
+        if (rules.find(rule_id) == rules.end()) {
+          // rules list must come before the edge definitions!
+          cerr << "Rule_id " << rule_id << " given but only loaded " << rules.size() << " rules\n";
+          return false;
+        }
+        cur_rule = rules[rule_id];
+      } else if (type == JSON_T_STRING) {
+        cur_rule.reset(new TRule(value->vu.str.value));
+      } else {
+        cerr << "Rule must be either a rule id or a rule string" << endl;
+        return false;
+      }
+      // cerr << "Edge: rule=" << cur_rule->AsString() << endl;
+      state = 15;
+      break;
+    case 17:      // edge.feats
+      if (type == JSON_T_NULL) { state = 15; break; }
+      assert(type == JSON_T_ARRAY_BEGIN);
+      state = 18; break;
+    case 18:
+      if (type == JSON_T_ARRAY_END) { state = 15; break; }
+      if (type != JSON_T_INTEGER && type != JSON_T_STRING) {
+        cerr << "Unexpected feature id type\n"; return false;
+      }
+      if (type == JSON_T_INTEGER) {
+        fid = value->vu.integer_value;
+        assert(fid < fdict.size());
+        fid = fdict[fid];
+      } else if (JSON_T_STRING) {
+        fid = FD::Convert(value->vu.str.value);
+      } else { abort(); }
+      state = 19;
+      break;
+    case 19:
+      {
+        assert(type == JSON_T_INTEGER || type == JSON_T_FLOAT);
+        double val = (type == JSON_T_INTEGER ? static_cast<double>(value->vu.integer_value) :
+	                                       strtod(value->vu.str.value, NULL));
+        feats.set_value(fid, val);
+        state = 18;
+        break;
+      }
+    case 20:     // edge.tail
+      if (type == JSON_T_NULL) { state = 15; break; }
+      assert(type == JSON_T_ARRAY_BEGIN);
+      state = 21; break;
+    case 21:
+      if (type == JSON_T_ARRAY_END) { state = 15; break; }
+      assert(type == JSON_T_INTEGER);
+      tail.push_back(value->vu.integer_value);
+      break;
+    case 22:     // edge.spans
+      assert(type == JSON_T_ARRAY_BEGIN);
+      state = 23;
+      spans[0] = spans[1] = spans[2] = spans[3] = -1;
+      spanc = 0;
+      break;
+    case 23:
+      if (type == JSON_T_ARRAY_END) { state = 15; break; }
+      assert(type == JSON_T_INTEGER);
+      assert(spanc < 4);
+      spans[spanc] = value->vu.integer_value;
+      ++spanc;
+    }
+    return true;
+  }
+  string rp;
+  string cat;
+  SmallVector tail;
+  vector<int> in_edges;
+  TRulePtr cur_rule;
+  map<int, TRulePtr> rules;
+  vector<int> fdict;
+  SparseVector<double> feats;
+  int state;
+  int fid;
+  int nodes;
+  int edges;
+  int spans[4];
+  int spanc;
+  string cur_key;
+  Hypergraph& hg;
+  int rule_id;
+  bool nodes_needed;
+  bool edges_needed;
+  bool is_sorted;
+};
+
+bool HypergraphIO::ReadFromJSON(istream* in, Hypergraph* hg) {
+  hg->clear();
+  HGReader reader(hg);
+  return reader.Parse(in);
+}
+
+static void WriteRule(const TRule& r, ostream* out) {
+  if (!r.lhs_) { (*out) << "[X] ||| "; }
+  JSONParser::WriteEscapedString(r.AsString(), out);
+}
+
+bool HypergraphIO::WriteToJSON(const Hypergraph& hg, bool remove_rules, ostream* out) {
+  map<const TRule*, int> rid;
+  ostream& o = *out;
+  rid[NULL] = 0;
+  o << '{';
+  if (!remove_rules) {
+    o << "\"rules\":[";
+    for (int i = 0; i < hg.edges_.size(); ++i) {
+      const TRule* r = hg.edges_[i].rule_.get();
+      int &id = rid[r];
+      if (!id) {
+        id=rid.size() - 1;
+        if (id > 1) o << ',';
+        o << id << ',';
+        WriteRule(*r, &o);
+      };
+    }
+    o << "],";
+  }
+  const bool use_fdict = FD::NumFeats() < 1000;
+  if (use_fdict) {
+    o << "\"features\":[";
+    for (int i = 1; i < FD::NumFeats(); ++i) {
+      o << (i==1 ? "":",") << '"' << FD::Convert(i) << '"';
+    }
+    o << "],";
+  }
+  vector<int> edgemap(hg.edges_.size(), -1);  // edges may be in non-topo order
+  int edge_count = 0;
+  for (int i = 0; i < hg.nodes_.size(); ++i) {
+    const Hypergraph::Node& node = hg.nodes_[i];
+    if (i > 0) { o << ","; }
+    o << "\"edges\":[";
+    for (int j = 0; j < node.in_edges_.size(); ++j) {
+      const Hypergraph::Edge& edge = hg.edges_[node.in_edges_[j]];
+      edgemap[edge.id_] = edge_count;
+      ++edge_count;
+      o << (j == 0 ? "" : ",") << "{";
+
+      o << "\"tail\":[";
+      for (int k = 0; k < edge.tail_nodes_.size(); ++k) {
+        o << (k > 0 ? "," : "") << edge.tail_nodes_[k];
+      }
+      o << "],";
+
+      o << "\"spans\":[" << edge.i_ << "," << edge.j_ << "," << edge.prev_i_ << "," << edge.prev_j_ << "],";
+
+      o << "\"feats\":[";
+      bool first = true;
+      for (SparseVector<double>::const_iterator it = edge.feature_values_.begin(); it != edge.feature_values_.end(); ++it) {
+        if (!it->second) continue;   // don't write features that have a zero value
+        if (!it->first) continue;    // if the feature set was frozen this might happen
+        if (!first) o << ',';
+        if (use_fdict)
+          o << (it->first - 1);
+        else
+          o << '"' << FD::Convert(it->first) << '"';
+        o << ',' << it->second;
+        first = false;
+      }
+      o << "]";
+      if (!remove_rules) { o << ",\"rule\":" << rid[edge.rule_.get()]; }
+      o << "}";
+    }
+    o << "],";
+
+    o << "\"node\":{\"in_edges\":[";
+    for (int j = 0; j < node.in_edges_.size(); ++j) {
+      int mapped_edge = edgemap[node.in_edges_[j]];
+      assert(mapped_edge >= 0);
+      o << (j == 0 ? "" : ",") << mapped_edge;
+    }
+    o << "]";
+    if (node.cat_ < 0) { o << ",\"cat\":\"" << TD::Convert(node.cat_ * -1) << '"'; }
+    o << "}";
+  }
+  o << "}\n";
+  return true;
+}
+
+bool needs_escape[128];
+void InitEscapes() {
+  memset(needs_escape, false, 128);
+  needs_escape[static_cast<size_t>('\'')] = true;
+  needs_escape[static_cast<size_t>('\\')] = true;
+}
+
+string HypergraphIO::Escape(const string& s) {
+  size_t len = s.size();
+  for (int i = 0; i < s.size(); ++i) {
+    unsigned char c = s[i];
+    if (c < 128 && needs_escape[c]) ++len;
+  }
+  if (len == s.size()) return s;
+  string res(len, ' ');
+  size_t o = 0;
+  for (int i = 0; i < s.size(); ++i) {
+    unsigned char c = s[i];
+    if (c < 128 && needs_escape[c])
+      res[o++] = '\\';
+    res[o++] = c;
+  }
+  assert(o == len);
+  return res;
+}
+
+string HypergraphIO::AsPLF(const Hypergraph& hg, bool include_global_parentheses) {
+  static bool first = true;
+  if (first) { InitEscapes(); first = false; }
+  if (hg.nodes_.empty()) return "()";
+  ostringstream os;
+  if (include_global_parentheses) os << '(';
+  static const string EPS="*EPS*";
+  for (int i = 0; i < hg.nodes_.size()-1; ++i) {
+    if (hg.nodes_[i].out_edges_.empty()) abort();
+    const bool last_node = (i == hg.nodes_.size() - 2);
+    const int out_edges_size = hg.nodes_[i].out_edges_.size();
+    // compound splitter adds an extra goal transition which we suppress with
+    // the following conditional
+    if (!last_node || out_edges_size != 1 ||
+         hg.edges_[hg.nodes_[i].out_edges_[0]].rule_->EWords() == 1) {
+      os << '(';
+      for (int j = 0; j < out_edges_size; ++j) {
+        const Hypergraph::Edge& e = hg.edges_[hg.nodes_[i].out_edges_[j]];
+        const string output = e.rule_->e_.size() ==2 ? Escape(TD::Convert(e.rule_->e_[1])) : EPS;
+        double prob = log(e.edge_prob_);
+        if (isinf(prob)) { prob = -9e20; }
+        if (isnan(prob)) { prob = 0; }
+        os << "('" << output << "'," << prob << "," << e.head_node_ - i << "),";
+      }
+      os << "),";
+    }
+  }
+  if (include_global_parentheses) os << ')';
+  return os.str();
+}
+
+namespace PLF {
+
+const string chars = "'\\";
+const char& quote = chars[0];
+const char& slash = chars[1];
+
+// safe get
+inline char get(const std::string& in, int c) {
+  if (c < 0 || c >= (int)in.size()) return 0;
+  else return in[(size_t)c];
+}
+
+// consume whitespace
+inline void eatws(const std::string& in, int& c) {
+  while (get(in,c) == ' ') { c++; }
+}
+
+// from 'foo' return foo
+std::string getEscapedString(const std::string& in, int &c)
+{
+  eatws(in,c);
+  if (get(in,c++) != quote) return "ERROR";
+  std::string res;
+  char cur = 0;
+  do {
+    cur = get(in,c++);
+    if (cur == slash) { res += get(in,c++); }
+    else if (cur != quote) { res += cur; }
+  } while (get(in,c) != quote && (c < (int)in.size()));
+  c++;
+  eatws(in,c);
+  return res;
+}
+
+// basically atof
+float getFloat(const std::string& in, int &c)
+{
+  std::string tmp;
+  eatws(in,c);
+  while (c < (int)in.size() && get(in,c) != ' ' && get(in,c) != ')' && get(in,c) != ',') {
+    tmp += get(in,c++);
+  }
+  eatws(in,c);
+  if (tmp.empty()) {
+    cerr << "Syntax error while reading number! col=" << c << endl;
+    abort();
+  }
+  return atof(tmp.c_str());
+}
+
+// basically atoi
+int getInt(const std::string& in, int &c)
+{
+  std::string tmp;
+  eatws(in,c);
+  while (c < (int)in.size() && get(in,c) != ' ' && get(in,c) != ')' && get(in,c) != ',') {
+    tmp += get(in,c++);
+  }
+  eatws(in,c);
+  return atoi(tmp.c_str());
+}
+
+// maximum number of nodes permitted
+#define MAX_NODES 100000000
+// parse ('foo', 0.23)
+void ReadPLFEdge(const std::string& in, int &c, int cur_node, Hypergraph* hg) {
+  if (get(in,c++) != '(') { assert(!"PCN/PLF parse error: expected ( at start of cn alt block\n"); }
+  vector<WordID> ewords(2, 0);
+  ewords[1] = TD::Convert(getEscapedString(in,c));
+  TRulePtr r(new TRule(ewords));
+  r->ComputeArity();
+  // cerr << "RULE: " << r->AsString() << endl;
+  if (get(in,c++) != ',') { cerr << in << endl; assert(!"PCN/PLF parse error: expected , after string\n"); }
+  size_t cnNext = 1;
+  std::vector<float> probs;
+  probs.push_back(getFloat(in,c));
+  while (get(in,c) == ',') {
+    c++;
+    float val = getFloat(in,c);
+    probs.push_back(val);
+    // cerr << val << endl;  //REMO
+  }
+  //if we read more than one prob, this was a lattice, last item was column increment
+  if (probs.size()>1) {
+    cnNext = static_cast<size_t>(probs.back());
+    probs.pop_back();
+    if (cnNext < 1) { cerr << cnNext << endl;
+             assert(!"PCN/PLF parse error: bad link length at last element of cn alt block\n"); }
+  }
+  if (get(in,c++) != ')') { assert(!"PCN/PLF parse error: expected ) at end of cn alt block\n"); }
+  eatws(in,c);
+  Hypergraph::TailNodeVector tail(1, cur_node);
+  Hypergraph::Edge* edge = hg->AddEdge(r, tail);
+  //cerr << "  <--" << cur_node << endl;
+  int head_node = cur_node + cnNext;
+  assert(head_node < MAX_NODES);  // prevent malicious PLFs from using all the memory
+  if (hg->nodes_.size() < (head_node + 1)) { hg->ResizeNodes(head_node + 1); }
+  hg->ConnectEdgeToHeadNode(edge, &hg->nodes_[head_node]);
+  for (int i = 0; i < probs.size(); ++i)
+    edge->feature_values_.set_value(FD::Convert("Feature_" + boost::lexical_cast<string>(i)), probs[i]);
+}
+
+// parse (('foo', 0.23), ('bar', 0.77))
+void ReadPLFNode(const std::string& in, int &c, int cur_node, int line, Hypergraph* hg) {
+  //cerr << "PLF READING NODE " << cur_node << endl;
+  if (hg->nodes_.size() < (cur_node + 1)) { hg->ResizeNodes(cur_node + 1); }
+  if (get(in,c++) != '(') { cerr << line << ": Syntax error 1\n"; abort(); }
+  eatws(in,c);
+  while (1) {
+    if (c > (int)in.size()) { break; }
+    if (get(in,c) == ')') {
+      c++;
+      eatws(in,c);
+      break;
+    }
+    if (get(in,c) == ',' && get(in,c+1) == ')') {
+      c+=2;
+      eatws(in,c);
+      break;
+    }
+    if (get(in,c) == ',') { c++; eatws(in,c); }
+    ReadPLFEdge(in, c, cur_node, hg);
+  }
+}
+
+} // namespace PLF 
+
+void HypergraphIO::ReadFromPLF(const std::string& in, Hypergraph* hg, int line) {
+  hg->clear();
+  int c = 0;
+  int cur_node = 0;
+  if (in[c++] != '(') { cerr << line << ": Syntax error!\n"; abort(); }
+  while (1) {
+    if (c > (int)in.size()) { break; }
+    if (PLF::get(in,c) == ')') {
+      c++;
+      PLF::eatws(in,c);
+      break;
+    }
+    if (PLF::get(in,c) == ',' && PLF::get(in,c+1) == ')') {
+      c+=2;
+      PLF::eatws(in,c);
+      break;
+    }
+    if (PLF::get(in,c) == ',') { c++; PLF::eatws(in,c); }
+    PLF::ReadPLFNode(in, c, cur_node, line, hg);
+    ++cur_node;
+  }
+  assert(cur_node == hg->nodes_.size() - 1);
+}
+
+void HypergraphIO::PLFtoLattice(const string& plf, Lattice* pl) {
+  Lattice& l = *pl;
+  Hypergraph g;
+  ReadFromPLF(plf, &g, 0);
+  const int num_nodes = g.nodes_.size() - 1;
+  l.resize(num_nodes);
+  for (int i = 0; i < num_nodes; ++i) {
+    vector<LatticeArc>& alts = l[i];
+    const Hypergraph::Node& node = g.nodes_[i];
+    const int num_alts = node.out_edges_.size();
+    alts.resize(num_alts);
+    for (int j = 0; j < num_alts; ++j) {
+      const Hypergraph::Edge& edge = g.edges_[node.out_edges_[j]];
+      alts[j].label = edge.rule_->e_[1];
+      alts[j].cost = edge.feature_values_.value(FD::Convert("Feature_0"));
+      alts[j].dist2next = edge.head_node_ - node.id_;
+    }
+  }
+}
+
+void HypergraphIO::WriteAsCFG(const Hypergraph& hg) {
+  vector<int> cats(hg.nodes_.size());
+  // each node in the translation forest becomes a "non-terminal" in the new
+  // grammar, create the labels here
+  const string kSEP = "_";
+  for (int i = 0; i < hg.nodes_.size(); ++i) {
+    const char* pstr = "CAT";
+    if (hg.nodes_[i].cat_ < 0)
+      pstr = TD::Convert(-hg.nodes_[i].cat_);
+    cats[i] = TD::Convert(pstr + kSEP + boost::lexical_cast<string>(i)) * -1;
+  }
+
+  for (int i = 0; i < hg.edges_.size(); ++i) {
+    const Hypergraph::Edge& edge = hg.edges_[i];
+    const vector<WordID>& tgt = edge.rule_->e();
+    const vector<WordID>& src = edge.rule_->f();
+    TRulePtr rule(new TRule);
+    rule->prev_i = edge.i_;
+    rule->prev_j = edge.j_;
+    rule->lhs_ = cats[edge.head_node_];
+    vector<WordID>& f = rule->f_;
+    vector<WordID>& e = rule->e_;
+    f.resize(tgt.size());   // swap source and target, since the parser
+    e.resize(src.size());   // parses using the source side!
+    Hypergraph::TailNodeVector tn(edge.tail_nodes_.size());
+    int ntc = 0;
+    for (int j = 0; j < tgt.size(); ++j) {
+      const WordID& cur = tgt[j];
+      if (cur > 0) {
+        f[j] = cur;
+      } else {
+        tn[ntc++] = cur;
+        f[j] = cats[edge.tail_nodes_[-cur]];
+      }
+    }
+    ntc = 0;
+    for (int j = 0; j < src.size(); ++j) {
+      const WordID& cur = src[j];
+      if (cur > 0) {
+        e[j] = cur;
+      } else {
+        e[j] = tn[ntc++];
+      }
+    }
+    rule->scores_ = edge.feature_values_;
+    rule->parent_rule_ = edge.rule_;
+    rule->ComputeArity();
+    cout << rule->AsString() << endl;
+  }
+}
+
+namespace B64 {
+
+static const char cb64[]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+static const char cd64[]="|$$$}rstuvwxyz{$$$$$$$>?@ABCDEFGHIJKLMNOPQRSTUVW$$$$$$XYZ[\\]^_`abcdefghijklmnopq";
+
+static void encodeblock(const unsigned char* in, ostream* os, int len) {
+  char out[4];
+  out[0] = cb64[ in[0] >> 2 ];
+  out[1] = cb64[ ((in[0] & 0x03) << 4) | ((in[1] & 0xf0) >> 4) ];
+  out[2] = (len > 1 ? cb64[ ((in[1] & 0x0f) << 2) | ((in[2] & 0xc0) >> 6) ] : '=');
+  out[3] = (len > 2 ? cb64[ in[2] & 0x3f ] : '=');
+  os->write(out, 4);
+}
+
+void b64encode(const char* data, const size_t size, ostream* out) {
+  size_t cur = 0;
+  while(cur < size) {
+    int len = min(static_cast<size_t>(3), size - cur);
+    encodeblock(reinterpret_cast<const unsigned char*>(&data[cur]), out, len);
+    cur += len;
+  }
+}
+
+static void decodeblock(const unsigned char* in, unsigned char* out) {   
+  out[0] = (unsigned char ) (in[0] << 2 | in[1] >> 4);
+  out[1] = (unsigned char ) (in[1] << 4 | in[2] >> 2);
+  out[2] = (unsigned char ) (((in[2] << 6) & 0xc0) | in[3]);
+}
+
+bool b64decode(const unsigned char* data, const size_t insize, char* out, const size_t outsize) {
+  size_t cur = 0;
+  size_t ocur = 0;
+  unsigned char in[4];
+  while(cur < insize) {
+    assert(ocur < outsize);
+    for (int i = 0; i < 4; ++i) {
+      unsigned char v = data[cur];
+      v = (unsigned char) ((v < 43 || v > 122) ? '\0' : cd64[ v - 43 ]);
+      if (!v) {
+        cerr << "B64 decode error at offset " << cur << " offending character: " << (int)data[cur] << endl;
+        return false;
+      }
+      v = (unsigned char) ((v == '$') ? '\0' : v - 61);
+      if (v) in[i] = v - 1; else in[i] = 0;
+      ++cur;
+    }
+    decodeblock(in, reinterpret_cast<unsigned char*>(&out[ocur]));
+    ocur += 3;
+  }
+  return true;
+}
+}
+
diff --git a/decoder/hg_io.h b/decoder/hg_io.h
new file mode 100644
index 00000000..7162106e
--- /dev/null
+++ b/decoder/hg_io.h
@@ -0,0 +1,39 @@
+#ifndef _HG_IO_H_
+#define _HG_IO_H_
+
+#include <iostream>
+
+#include "lattice.h"
+class Hypergraph;
+
+struct HypergraphIO {
+
+  // the format is basically a list of nodes and edges in topological order
+  // any edge you read, you must have already read its tail nodes
+  // any node you read, you must have already read its incoming edges
+  // this may make writing a bit more challenging if your forest is not
+  // topologically sorted (but that probably doesn't happen very often),
+  // but it makes reading much more memory efficient.
+  // see test_data/small.json.gz for an email encoding
+  static bool ReadFromJSON(std::istream* in, Hypergraph* out);
+
+  // if remove_rules is used, the hypergraph is serialized without rule information
+  // (so it only contains structure and feature information)
+  static bool WriteToJSON(const Hypergraph& hg, bool remove_rules, std::ostream* out);
+
+  static void WriteAsCFG(const Hypergraph& hg);
+
+  // serialization utils
+  static void ReadFromPLF(const std::string& in, Hypergraph* out, int line = 0);
+  // return PLF string representation (undefined behavior on non-lattices)
+  static std::string AsPLF(const Hypergraph& hg, bool include_global_parentheses = true);
+  static void PLFtoLattice(const std::string& plf, Lattice* pl);
+  static std::string Escape(const std::string& s);  // PLF helper
+};
+
+namespace B64 {
+  bool b64decode(const unsigned char* data, const size_t insize, char* out, const size_t outsize);
+  void b64encode(const char* data, const size_t size, std::ostream* out);
+}
+
+#endif
diff --git a/decoder/hg_test.cc b/decoder/hg_test.cc
new file mode 100644
index 00000000..51934ad1
--- /dev/null
+++ b/decoder/hg_test.cc
@@ -0,0 +1,455 @@
+#include <cassert>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <gtest/gtest.h>
+#include "tdict.h"
+
+#include "json_parse.h"
+#include "filelib.h"
+#include "hg.h"
+#include "hg_io.h"
+#include "hg_intersect.h"
+#include "viterbi.h"
+#include "kbest.h"
+#include "inside_outside.h"
+
+using namespace std;
+
+class HGTest : public testing::Test {
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+  void CreateHG(Hypergraph* hg) const;
+  void CreateHG_int(Hypergraph* hg) const;
+  void CreateHG_tiny(Hypergraph* hg) const;
+  void CreateHGBalanced(Hypergraph* hg) const;
+  void CreateLatticeHG(Hypergraph* hg) const;
+  void CreateTinyLatticeHG(Hypergraph* hg) const;
+};
+       
+void HGTest::CreateTinyLatticeHG(Hypergraph* hg) const {
+  const string json = "{\"rules\":[1,\"[X] ||| [1] a\",2,\"[X] ||| [1] A\",3,\"[X] ||| [1] b\",4,\"[X] ||| [1] B'\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[],\"node\":{\"in_edges\":[]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.2],\"rule\":1},{\"tail\":[0],\"feats\":[0,-0.6],\"rule\":2}],\"node\":{\"in_edges\":[0,1]},\"edges\":[{\"tail\":[1],\"feats\":[0,-0.1],\"rule\":3},{\"tail\":[1],\"feats\":[0,-0.9],\"rule\":4}],\"node\":{\"in_edges\":[2,3]}}";
+  istringstream instr(json);
+  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
+}
+
+void HGTest::CreateLatticeHG(Hypergraph* hg) const {
+  const string json = "{\"rules\":[1,\"[X] ||| [1] a\",2,\"[X] ||| [1] A\",3,\"[X] ||| [1] A A\",4,\"[X] ||| [1] b\",5,\"[X] ||| [1] c\",6,\"[X] ||| [1] B C\",7,\"[X] ||| [1] A B C\",8,\"[X] ||| [1] CC\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[],\"node\":{\"in_edges\":[]},\"edges\":[{\"tail\":[0],\"feats\":[2,-0.3],\"rule\":1},{\"tail\":[0],\"feats\":[2,-0.6],\"rule\":2},{\"tail\":[0],\"feats\":[2,-1.7],\"rule\":3}],\"node\":{\"in_edges\":[0,1,2]},\"edges\":[{\"tail\":[1],\"feats\":[2,-0.5],\"rule\":4}],\"node\":{\"in_edges\":[3]},\"edges\":[{\"tail\":[2],\"feats\":[2,-0.6],\"rule\":5},{\"tail\":[1],\"feats\":[2,-0.8],\"rule\":6},{\"tail\":[0],\"feats\":[2,-0.01],\"rule\":7},{\"tail\":[2],\"feats\":[2,-0.8],\"rule\":8}],\"node\":{\"in_edges\":[4,5,6,7]}}";
+  istringstream instr(json);
+  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
+}
+
+void HGTest::CreateHG_tiny(Hypergraph* hg) const {
+  const string json = "{\"rules\":[1,\"[X] ||| <s>\",2,\"[X] ||| X [1]\",3,\"[X] ||| Z [1]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[0,-2,1,-99],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.5,1,-0.8],\"rule\":2},{\"tail\":[0],\"feats\":[0,-0.7,1,-0.9],\"rule\":3}],\"node\":{\"in_edges\":[1,2]}}";
+  istringstream instr(json);
+  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
+}
+
+void HGTest::CreateHG_int(Hypergraph* hg) const {
+  const string json = "{\"rules\":[1,\"[X] ||| a\",2,\"[X] ||| b\",3,\"[X] ||| a [1]\",4,\"[X] ||| [1] b\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[0,0.1],\"rule\":1},{\"tail\":[],\"feats\":[0,0.1],\"rule\":2}],\"node\":{\"in_edges\":[0,1],\"cat\":\"X\"},\"edges\":[{\"tail\":[0],\"feats\":[0,0.3],\"rule\":3},{\"tail\":[0],\"feats\":[0,0.2],\"rule\":4}],\"node\":{\"in_edges\":[2,3],\"cat\":\"Goal\"}}";
+  istringstream instr(json);
+  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
+}
+
+void HGTest::CreateHG(Hypergraph* hg) const {
+  string json = "{\"rules\":[1,\"[X] ||| a\",2,\"[X] ||| A [1]\",3,\"[X] ||| c\",4,\"[X] ||| C [1]\",5,\"[X] ||| [1] B [2]\",6,\"[X] ||| [1] b [2]\",7,\"[X] ||| X [1]\",8,\"[X] ||| Z [1]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.8,1,-0.1],\"rule\":2}],\"node\":{\"in_edges\":[1]},\"edges\":[{\"tail\":[],\"feats\":[1,-1],\"rule\":3}],\"node\":{\"in_edges\":[2]},\"edges\":[{\"tail\":[2],\"feats\":[0,-0.2,1,-0.1],\"rule\":4}],\"node\":{\"in_edges\":[3]},\"edges\":[{\"tail\":[1,3],\"feats\":[0,-1.2,1,-0.2],\"rule\":5},{\"tail\":[1,3],\"feats\":[0,-0.5,1,-1.3],\"rule\":6}],\"node\":{\"in_edges\":[4,5]},\"edges\":[{\"tail\":[4],\"feats\":[0,-0.5,1,-0.8],\"rule\":7},{\"tail\":[4],\"feats\":[0,-0.7,1,-0.9],\"rule\":8}],\"node\":{\"in_edges\":[6,7]}}";
+  istringstream instr(json);
+  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
+}
+
+void HGTest::CreateHGBalanced(Hypergraph* hg) const {
+  const string json = "{\"rules\":[1,\"[X] ||| i\",2,\"[X] ||| a\",3,\"[X] ||| b\",4,\"[X] ||| [1] [2]\",5,\"[X] ||| [1] [2]\",6,\"[X] ||| c\",7,\"[X] ||| d\",8,\"[X] ||| [1] [2]\",9,\"[X] ||| [1] [2]\",10,\"[X] ||| [1] [2]\",11,\"[X] ||| [1] [2]\",12,\"[X] ||| [1] [2]\",13,\"[X] ||| [1] [2]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":2}],\"node\":{\"in_edges\":[1]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":3}],\"node\":{\"in_edges\":[2]},\"edges\":[{\"tail\":[1,2],\"feats\":[],\"rule\":4},{\"tail\":[2,1],\"feats\":[],\"rule\":5}],\"node\":{\"in_edges\":[3,4]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":6}],\"node\":{\"in_edges\":[5]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":7}],\"node\":{\"in_edges\":[6]},\"edges\":[{\"tail\":[4,5],\"feats\":[],\"rule\":8},{\"tail\":[5,4],\"feats\":[],\"rule\":9}],\"node\":{\"in_edges\":[7,8]},\"edges\":[{\"tail\":[3,6],\"feats\":[],\"rule\":10},{\"tail\":[6,3],\"feats\":[],\"rule\":11}],\"node\":{\"in_edges\":[9,10]},\"edges\":[{\"tail\":[7,0],\"feats\":[],\"rule\":12},{\"tail\":[0,7],\"feats\":[],\"rule\":13}],\"node\":{\"in_edges\":[11,12]}}";
+  istringstream instr(json);
+  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
+}
+
+TEST_F(HGTest,Controlled) {
+  Hypergraph hg;
+  CreateHG_tiny(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 0.4);
+  wts.set_value(FD::Convert("f2"), 0.8);
+  hg.Reweight(wts);
+  vector<WordID> trans;
+  prob_t prob = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "prob: " << prob << "\n";
+  EXPECT_FLOAT_EQ(-80.839996, log(prob));
+  EXPECT_EQ("X <s>", TD::GetString(trans));
+  vector<prob_t> post;
+  hg.PrintGraphviz();
+  prob_t c2 = Inside<prob_t, ScaledEdgeProb>(hg, NULL, ScaledEdgeProb(0.6));
+  EXPECT_FLOAT_EQ(-47.8577, log(c2));
+}
+
+TEST_F(HGTest,Union) {
+  Hypergraph hg1;
+  Hypergraph hg2;
+  CreateHG_tiny(&hg1);
+  CreateHG(&hg2);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 0.4);
+  wts.set_value(FD::Convert("f2"), 1.0);
+  hg1.Reweight(wts);
+  hg2.Reweight(wts);
+  prob_t c1,c2,c3,c4;
+  vector<WordID> t1,t2,t3,t4;
+  c1 = ViterbiESentence(hg1, &t1);
+  c2 = ViterbiESentence(hg2, &t2);
+  int l2 = ViterbiPathLength(hg2);
+  cerr << c1 << "\t" << TD::GetString(t1) << endl;
+  cerr << c2 << "\t" << TD::GetString(t2) << endl;
+  hg1.Union(hg2);
+  hg1.Reweight(wts);
+  c3 = ViterbiESentence(hg1, &t3);
+  int l3 = ViterbiPathLength(hg1);
+  cerr << c3 << "\t" << TD::GetString(t3) << endl;
+  EXPECT_FLOAT_EQ(c2, c3);
+  EXPECT_EQ(TD::GetString(t2), TD::GetString(t3));
+  EXPECT_EQ(l2, l3);
+
+  wts.set_value(FD::Convert("f2"), -1);
+  hg1.Reweight(wts);
+  c4 = ViterbiESentence(hg1, &t4);
+  cerr << c4 << "\t" << TD::GetString(t4) << endl;
+  EXPECT_EQ("Z <s>", TD::GetString(t4));
+  EXPECT_FLOAT_EQ(98.82, log(c4));
+
+  vector<pair<vector<WordID>, prob_t> > list;
+  KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(hg1, 10);
+  for (int i = 0; i < 10; ++i) {
+    const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
+      kbest.LazyKthBest(hg1.nodes_.size() - 1, i);
+    if (!d) break;
+    list.push_back(make_pair(d->yield, d->score));
+  }
+  EXPECT_TRUE(list[0].first == t4);
+  EXPECT_FLOAT_EQ(log(list[0].second), log(c4));
+  EXPECT_EQ(list.size(), 6);
+  EXPECT_FLOAT_EQ(log(list.back().second / list.front().second), -97.7);
+}
+
+TEST_F(HGTest,ControlledKBest) {
+  Hypergraph hg;
+  CreateHG(&hg);
+  vector<double> w(2); w[0]=0.4; w[1]=0.8;
+  hg.Reweight(w);
+  vector<WordID> trans;
+  prob_t cost = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "cost: " << cost << "\n";
+
+  int best = 0;
+  KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(hg, 10);
+  for (int i = 0; i < 10; ++i) {
+    const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
+      kbest.LazyKthBest(hg.nodes_.size() - 1, i);
+    if (!d) break;
+    cerr << TD::GetString(d->yield) << endl;
+    ++best;
+  }
+  EXPECT_EQ(4, best);
+}
+
+
+TEST_F(HGTest,InsideScore) {
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 1.0);
+  Hypergraph hg;
+  CreateTinyLatticeHG(&hg);
+  hg.Reweight(wts);
+  vector<WordID> trans;
+  prob_t cost = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "cost: " << cost << "\n";
+  hg.PrintGraphviz();
+  prob_t inside = Inside<prob_t, EdgeProb>(hg);
+  EXPECT_FLOAT_EQ(1.7934048, inside);  // computed by hand
+  vector<prob_t> post;
+  inside = hg.ComputeBestPathThroughEdges(&post);
+  EXPECT_FLOAT_EQ(-0.3, log(inside));  // computed by hand
+  EXPECT_EQ(post.size(), 4);
+  for (int i = 0; i < 4; ++i) {
+    cerr << "edge post: " << log(post[i]) << '\t' << hg.edges_[i].rule_->AsString() << endl;
+  }
+}
+
+
+TEST_F(HGTest,PruneInsideOutside) {
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("Feature_1"), 1.0);
+  Hypergraph hg;
+  CreateLatticeHG(&hg);
+  hg.Reweight(wts);
+  vector<WordID> trans;
+  prob_t cost = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "cost: " << cost << "\n";
+  hg.PrintGraphviz();
+  //hg.DensityPruneInsideOutside(0.5, false, 2.0);
+  hg.BeamPruneInsideOutside(0.5, false, 0.5);
+  cost = ViterbiESentence(hg, &trans);
+  cerr << "Ncst: " << cost << endl;
+  cerr << TD::GetString(trans) << "\n";
+  hg.PrintGraphviz();
+}
+
+TEST_F(HGTest,TestPruneEdges) {
+  Hypergraph hg;
+  CreateLatticeHG(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 1.0);
+  hg.Reweight(wts);
+  hg.PrintGraphviz();
+  vector<bool> prune(hg.edges_.size(), true);
+  prune[6] = false;
+  hg.PruneEdges(prune);
+  cerr << "Pruned:\n";
+  hg.PrintGraphviz();
+}
+
+TEST_F(HGTest,TestIntersect) {
+  Hypergraph hg;
+  CreateHG_int(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 1.0);
+  hg.Reweight(wts);
+  hg.PrintGraphviz();
+
+  int best = 0;
+  KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(hg, 10);
+  for (int i = 0; i < 10; ++i) {
+    const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
+      kbest.LazyKthBest(hg.nodes_.size() - 1, i);
+    if (!d) break;
+    cerr << TD::GetString(d->yield) << endl;
+    ++best;
+  }
+  EXPECT_EQ(4, best);
+
+  Lattice target(2);
+  target[0].push_back(LatticeArc(TD::Convert("a"), 0.0, 1));
+  target[1].push_back(LatticeArc(TD::Convert("b"), 0.0, 1));
+  HG::Intersect(target, &hg);
+  hg.PrintGraphviz();
+}
+
+TEST_F(HGTest,TestPrune2) {
+  Hypergraph hg;
+  CreateHG_int(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 1.0);
+  hg.Reweight(wts);
+  hg.PrintGraphviz();
+  vector<bool> rem(hg.edges_.size(), false);
+  rem[0] = true;
+  rem[1] = true;
+  hg.PruneEdges(rem);
+  hg.PrintGraphviz();
+  cerr << "TODO: fix this pruning behavior-- the resulting HG should be empty!\n";
+}
+
+TEST_F(HGTest,Sample) {
+  Hypergraph hg;
+  CreateLatticeHG(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("Feature_1"), 0.0);
+  hg.Reweight(wts);
+  vector<WordID> trans;
+  prob_t cost = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "cost: " << cost << "\n";
+  hg.PrintGraphviz();
+}
+
+TEST_F(HGTest,PLF) {
+  Hypergraph hg;
+  string inplf = "((('haupt',-2.06655,1),('hauptgrund',-5.71033,2),),(('grund',-1.78709,1),),(('für\\'',0.1,1),),)";
+  HypergraphIO::ReadFromPLF(inplf, &hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("Feature_0"), 1.0);
+  hg.Reweight(wts);
+  hg.PrintGraphviz();
+  string outplf = HypergraphIO::AsPLF(hg);
+  cerr << " IN: " << inplf << endl;
+  cerr << "OUT: " << outplf << endl;
+  assert(inplf == outplf);
+}
+
+TEST_F(HGTest,PushWeightsToGoal) {
+  Hypergraph hg;
+  CreateHG(&hg);
+  vector<double> w(2); w[0]=0.4; w[1]=0.8;
+  hg.Reweight(w);
+  vector<WordID> trans;
+  prob_t cost = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "cost: " << cost << "\n";
+  hg.PrintGraphviz();
+  hg.PushWeightsToGoal();
+  hg.PrintGraphviz();
+}
+
+TEST_F(HGTest,TestSpecialKBest) {
+  Hypergraph hg;
+  CreateHGBalanced(&hg);
+  vector<double> w(1); w[0]=0;
+  hg.Reweight(w);
+  vector<pair<vector<WordID>, prob_t> > list;
+  KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(hg, 100000);
+  for (int i = 0; i < 100000; ++i) {
+    const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
+      kbest.LazyKthBest(hg.nodes_.size() - 1, i);
+    if (!d) break;
+    cerr << TD::GetString(d->yield) << endl;
+  }
+  hg.PrintGraphviz();
+}
+
+TEST_F(HGTest, TestGenericViterbi) {
+  Hypergraph hg;
+  CreateHG_tiny(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 0.4);
+  wts.set_value(FD::Convert("f2"), 0.8);
+  hg.Reweight(wts);
+  vector<WordID> trans;
+  const prob_t prob = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "prob: " << prob << "\n";
+  EXPECT_FLOAT_EQ(-80.839996, log(prob));
+  EXPECT_EQ("X <s>", TD::GetString(trans));
+}
+
+TEST_F(HGTest, TestGenericInside) {
+  Hypergraph hg;
+  CreateTinyLatticeHG(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 1.0);
+  hg.Reweight(wts);
+  vector<prob_t> inside;
+  prob_t ins = Inside<prob_t, EdgeProb>(hg, &inside);
+  EXPECT_FLOAT_EQ(1.7934048, ins);  // computed by hand
+  vector<prob_t> outside;
+  Outside<prob_t, EdgeProb>(hg, inside, &outside);
+  EXPECT_EQ(3, outside.size());
+  EXPECT_FLOAT_EQ(1.7934048, outside[0]);
+  EXPECT_FLOAT_EQ(1.3114071, outside[1]);
+  EXPECT_FLOAT_EQ(1.0, outside[2]);
+}
+
+TEST_F(HGTest,TestGenericInside2) {
+  Hypergraph hg;
+  CreateHG(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 0.4);
+  wts.set_value(FD::Convert("f2"), 0.8);
+  hg.Reweight(wts);
+  vector<prob_t> inside, outside;
+  prob_t ins = Inside<prob_t, EdgeProb>(hg, &inside);
+  Outside<prob_t, EdgeProb>(hg, inside, &outside);
+  for (int i = 0; i < hg.nodes_.size(); ++i)
+    cerr << i << "\t" << log(inside[i]) << "\t" << log(outside[i]) << endl;
+  EXPECT_FLOAT_EQ(0, log(inside[0]));
+  EXPECT_FLOAT_EQ(-1.7861683, log(outside[0]));
+  EXPECT_FLOAT_EQ(-0.4, log(inside[1]));
+  EXPECT_FLOAT_EQ(-1.3861683, log(outside[1]));
+  EXPECT_FLOAT_EQ(-0.8, log(inside[2]));
+  EXPECT_FLOAT_EQ(-0.986168, log(outside[2]));
+  EXPECT_FLOAT_EQ(-0.96, log(inside[3]));
+  EXPECT_FLOAT_EQ(-0.8261683, log(outside[3]));
+  EXPECT_FLOAT_EQ(-1.562512, log(inside[4]));
+  EXPECT_FLOAT_EQ(-0.22365622, log(outside[4]));
+  EXPECT_FLOAT_EQ(-1.7861683, log(inside[5]));
+  EXPECT_FLOAT_EQ(0, log(outside[5]));
+}
+
+TEST_F(HGTest,TestAddExpectations) {
+  Hypergraph hg;
+  CreateHG(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 0.4);
+  wts.set_value(FD::Convert("f2"), 0.8);
+  hg.Reweight(wts);
+  SparseVector<prob_t> feat_exps;
+  prob_t z = InsideOutside<prob_t, EdgeProb,
+                  SparseVector<prob_t>, EdgeFeaturesAndProbWeightFunction>(hg, &feat_exps);
+  EXPECT_FLOAT_EQ(-2.5439765, feat_exps[FD::Convert("f1")] / z);
+  EXPECT_FLOAT_EQ(-2.6357865, feat_exps[FD::Convert("f2")] / z);
+  cerr << feat_exps << endl;
+  cerr << "Z=" << z << endl;
+}
+
+TEST_F(HGTest, Small) {
+  ReadFile rf("test_data/small.json.gz");
+  Hypergraph hg;
+  assert(HypergraphIO::ReadFromJSON(rf.stream(), &hg));
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("Model_0"), -2.0);
+  wts.set_value(FD::Convert("Model_1"), -0.5);
+  wts.set_value(FD::Convert("Model_2"), -1.1);
+  wts.set_value(FD::Convert("Model_3"), -1.0);
+  wts.set_value(FD::Convert("Model_4"), -1.0);
+  wts.set_value(FD::Convert("Model_5"), 0.5);
+  wts.set_value(FD::Convert("Model_6"), 0.2);
+  wts.set_value(FD::Convert("Model_7"), -3.0);
+  hg.Reweight(wts);
+  vector<WordID> trans;
+  prob_t cost = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "cost: " << cost << "\n";
+  vector<prob_t> post;
+  prob_t c2 = Inside<prob_t, ScaledEdgeProb>(hg, NULL, ScaledEdgeProb(0.6));
+  EXPECT_FLOAT_EQ(2.1431036, log(c2));
+}
+
+TEST_F(HGTest, JSONTest) {
+  ostringstream os;
+  JSONParser::WriteEscapedString("\"I don't know\", she said.", &os);
+  EXPECT_EQ("\"\\\"I don't know\\\", she said.\"", os.str());
+  ostringstream os2;
+  JSONParser::WriteEscapedString("yes", &os2);
+  EXPECT_EQ("\"yes\"", os2.str());
+}
+
+TEST_F(HGTest, TestGenericKBest) {
+  Hypergraph hg;
+  CreateHG(&hg);
+  //CreateHGBalanced(&hg);
+  SparseVector<double> wts;
+  wts.set_value(FD::Convert("f1"), 0.4);
+  wts.set_value(FD::Convert("f2"), 1.0);
+  hg.Reweight(wts);
+  vector<WordID> trans;
+  prob_t cost = ViterbiESentence(hg, &trans);
+  cerr << TD::GetString(trans) << "\n";
+  cerr << "cost: " << cost << "\n";
+
+  KBest::KBestDerivations<vector<WordID>, ESentenceTraversal> kbest(hg, 1000);
+  for (int i = 0; i < 1000; ++i) {
+    const KBest::KBestDerivations<vector<WordID>, ESentenceTraversal>::Derivation* d =
+      kbest.LazyKthBest(hg.nodes_.size() - 1, i);
+    if (!d) break;
+    cerr << TD::GetString(d->yield) << " F:" << d->feature_values << endl;
+  }
+}
+
+TEST_F(HGTest, TestReadWriteHG) {
+  Hypergraph hg,hg2;
+  CreateHG(&hg);
+  hg.edges_.front().j_ = 23;
+  hg.edges_.back().prev_i_ = 99;
+  ostringstream os;
+  HypergraphIO::WriteToJSON(hg, false, &os);
+  istringstream is(os.str());
+  HypergraphIO::ReadFromJSON(&is, &hg2);
+  EXPECT_EQ(hg2.NumberOfPaths(), hg.NumberOfPaths());
+  EXPECT_EQ(hg2.edges_.front().j_, 23);
+  EXPECT_EQ(hg2.edges_.back().prev_i_, 99);
+}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/decoder/inside_outside.h b/decoder/inside_outside.h
new file mode 100644
index 00000000..3c7518f2
--- /dev/null
+++ b/decoder/inside_outside.h
@@ -0,0 +1,112 @@
+#ifndef _INSIDE_H_
+#define _INSIDE_H_
+
+#include <vector>
+#include <algorithm>
+#include "hg.h"
+
+// run the inside algorithm and return the inside score
+// if result is non-NULL, result will contain the inside
+// score for each node
+// NOTE: WeightType()  must construct the semiring's additive identity
+//       WeightType(1) must construct the semiring's multiplicative identity
+template<typename WeightType, typename WeightFunction>
+WeightType Inside(const Hypergraph& hg,
+                  std::vector<WeightType>* result = NULL,
+                  const WeightFunction& weight = WeightFunction()) {
+  const int num_nodes = hg.nodes_.size();
+  std::vector<WeightType> dummy;
+  std::vector<WeightType>& inside_score = result ? *result : dummy;
+  inside_score.resize(num_nodes);
+  std::fill(inside_score.begin(), inside_score.end(), WeightType());
+  for (int i = 0; i < num_nodes; ++i) {
+    const Hypergraph::Node& cur_node = hg.nodes_[i];
+    WeightType* const cur_node_inside_score = &inside_score[i];
+    const int num_in_edges = cur_node.in_edges_.size();
+    if (num_in_edges == 0) {
+      *cur_node_inside_score = WeightType(1);
+      continue;
+    }
+    for (int j = 0; j < num_in_edges; ++j) {
+      const Hypergraph::Edge& edge = hg.edges_[cur_node.in_edges_[j]];
+      WeightType score = weight(edge);
+      for (int k = 0; k < edge.tail_nodes_.size(); ++k) {
+        const int tail_node_index = edge.tail_nodes_[k];
+        score *= inside_score[tail_node_index];
+      }
+      *cur_node_inside_score += score;
+    }
+  }
+  return inside_score.back();
+}
+
+template<typename WeightType, typename WeightFunction>
+void Outside(const Hypergraph& hg,
+             std::vector<WeightType>& inside_score,
+             std::vector<WeightType>* result,
+             const WeightFunction& weight = WeightFunction()) {
+  assert(result);
+  const int num_nodes = hg.nodes_.size();
+  assert(inside_score.size() == num_nodes);
+  std::vector<WeightType>& outside_score = *result;
+  outside_score.resize(num_nodes);
+  std::fill(outside_score.begin(), outside_score.end(), WeightType());
+  outside_score.back() = WeightType(1);
+  for (int i = num_nodes - 1; i >= 0; --i) {
+    const Hypergraph::Node& cur_node = hg.nodes_[i];
+    const WeightType& head_node_outside_score = outside_score[i];
+    const int num_in_edges = cur_node.in_edges_.size();
+    for (int j = 0; j < num_in_edges; ++j) {
+      const Hypergraph::Edge& edge = hg.edges_[cur_node.in_edges_[j]];
+      WeightType head_and_edge_weight = weight(edge);
+      head_and_edge_weight *= head_node_outside_score;
+      const int num_tail_nodes = edge.tail_nodes_.size();
+      for (int k = 0; k < num_tail_nodes; ++k) {
+        const int update_tail_node_index = edge.tail_nodes_[k];
+        WeightType* const tail_outside_score = &outside_score[update_tail_node_index];
+        WeightType inside_contribution = WeightType(1);
+        for (int l = 0; l < num_tail_nodes; ++l) {
+          const int other_tail_node_index = edge.tail_nodes_[l];
+          if (update_tail_node_index != other_tail_node_index)
+            inside_contribution *= inside_score[other_tail_node_index];
+        }
+        inside_contribution *= head_and_edge_weight;
+        *tail_outside_score += inside_contribution;
+      }
+    }
+  }
+}
+
+// this is the Inside-Outside optimization described in Li and Eisner (EMNLP 2009)
+// for computing the inside algorithm over expensive semirings
+// (such as expectations over features).  See Figure 4.
+// NOTE: XType * KType must be valid (and yield XType)
+// NOTE: This may do things slightly differently than you are used to, please
+// read the description in Li and Eisner (2009) carefully!
+template<typename KType, typename KWeightFunction, typename XType, typename XWeightFunction>
+KType InsideOutside(const Hypergraph& hg,
+                    XType* result_x,
+                    const KWeightFunction& kwf = KWeightFunction(),
+                    const XWeightFunction& xwf = XWeightFunction()) {
+  const int num_nodes = hg.nodes_.size();
+  std::vector<KType> inside, outside;
+  const KType k = Inside<KType,KWeightFunction>(hg, &inside, kwf);
+  Outside<KType,KWeightFunction>(hg, inside, &outside, kwf);
+  XType& x = *result_x;
+  x = XType();      // default constructor is semiring 0
+  for (int i = 0; i < num_nodes; ++i) {
+    const Hypergraph::Node& cur_node = hg.nodes_[i];
+    const int num_in_edges = cur_node.in_edges_.size();
+    for (int j = 0; j < num_in_edges; ++j) {
+      const Hypergraph::Edge& edge = hg.edges_[cur_node.in_edges_[j]];
+      KType kbar_e = outside[i];
+      const int num_tail_nodes = edge.tail_nodes_.size();
+      for (int k = 0; k < num_tail_nodes; ++k)
+        kbar_e *= inside[edge.tail_nodes_[k]];
+      x += xwf(edge) * kbar_e;
+    }
+  }
+  return k;
+}
+
+#endif
diff --git a/decoder/json_parse.cc b/decoder/json_parse.cc
new file mode 100644
index 00000000..f6fdfea8
--- /dev/null
+++ b/decoder/json_parse.cc
@@ -0,0 +1,50 @@
+#include "json_parse.h"
+
+#include <string>
+#include <iostream>
+
+using namespace std;
+
+static const char *json_hex_chars = "0123456789abcdef";
+
+void JSONParser::WriteEscapedString(const string& in, ostream* out) {
+  int pos = 0;
+  int start_offset = 0;
+  unsigned char c = 0;
+  (*out) << '"';
+  while(pos < in.size()) {
+    c = in[pos];
+    switch(c) {
+    case '\b':
+    case '\n':
+    case '\r':
+    case '\t':
+    case '"':
+    case '\\':
+    case '/':
+      if(pos - start_offset > 0)
+	(*out) << in.substr(start_offset, pos - start_offset);
+      if(c == '\b') (*out) << "\\b";
+      else if(c == '\n') (*out) << "\\n";
+      else if(c == '\r') (*out) << "\\r";
+      else if(c == '\t') (*out) << "\\t";
+      else if(c == '"') (*out) << "\\\"";
+      else if(c == '\\') (*out) << "\\\\";
+      else if(c == '/') (*out) << "\\/";
+      start_offset = ++pos;
+      break;
+    default:
+      if(c < ' ') {
+        cerr << "Warning, bad character (" << static_cast<int>(c) << ") in string\n";
+	if(pos - start_offset > 0)
+	  (*out) << in.substr(start_offset, pos - start_offset);
+	(*out) << "\\u00" << json_hex_chars[c >> 4] << json_hex_chars[c & 0xf];
+	start_offset = ++pos;
+      } else pos++;
+    }
+  }
+  if(pos - start_offset > 0)
+    (*out) << in.substr(start_offset, pos - start_offset);
+  (*out) << '"';
+}
+
diff --git a/decoder/json_parse.h b/decoder/json_parse.h
new file mode 100644
index 00000000..c3cba954
--- /dev/null
+++ b/decoder/json_parse.h
@@ -0,0 +1,58 @@
+#ifndef _JSON_WRAPPER_H_
+#define _JSON_WRAPPER_H_
+
+#include <iostream>
+#include <cassert>
+#include "JSON_parser.h"
+
+class JSONParser {
+ public:
+  JSONParser() {
+    init_JSON_config(&config);
+    hack.mf = &JSONParser::Callback;
+    config.depth = 10;
+    config.callback_ctx = reinterpret_cast<void*>(this);
+    config.callback = hack.cb;
+    config.allow_comments = 1;
+    config.handle_floats_manually = 1;
+    jc = new_JSON_parser(&config);
+  }
+  virtual ~JSONParser() {
+    delete_JSON_parser(jc);
+  }
+  bool Parse(std::istream* in) {
+    int count = 0;
+    int lc = 1;
+    for (; in ; ++count) {
+      int next_char = in->get();
+      if (!in->good()) break;
+      if (lc == '\n') { ++lc; }
+      if (!JSON_parser_char(jc, next_char)) {
+        std::cerr << "JSON_parser_char: syntax error, line " << lc << " (byte " << count << ")" << std::endl;
+        return false;
+      }
+    }
+    if (!JSON_parser_done(jc)) {
+      std::cerr << "JSON_parser_done: syntax error\n";
+      return false;
+    }
+    return true;
+  }
+  static void WriteEscapedString(const std::string& in, std::ostream* out);
+ protected:
+  virtual bool HandleJSONEvent(int type, const JSON_value* value) = 0;
+ private:
+  int Callback(int type, const JSON_value* value) {
+    if (HandleJSONEvent(type, value)) return 1;
+    return 0;
+  }
+  JSON_parser_struct* jc;
+  JSON_config config;
+  typedef int (JSONParser::* MF)(int type, const struct JSON_value_struct* value);
+  union CBHack {
+    JSON_parser_callback cb;
+    MF mf;
+  } hack;
+};
+
+#endif
diff --git a/decoder/kbest.h b/decoder/kbest.h
new file mode 100644
index 00000000..fcd40fcd
--- /dev/null
+++ b/decoder/kbest.h
@@ -0,0 +1,208 @@
+#ifndef _HG_KBEST_H_
+#define _HG_KBEST_H_
+
+#include <vector>
+#include <utility>
+#include <tr1/unordered_set>
+
+#include <boost/shared_ptr.hpp>
+
+#include "wordid.h"
+#include "hg.h"
+
+namespace KBest {
+  // default, don't filter any derivations from the k-best list
+  struct NoFilter {
+    bool operator()(const std::vector<WordID>& yield) {
+      (void) yield;
+      return false;
+    }
+  };
+
+  // optional, filter unique yield strings
+  struct FilterUnique {
+    std::tr1::unordered_set<std::vector<WordID>, boost::hash<std::vector<WordID> > > unique;
+
+    bool operator()(const std::vector<WordID>& yield) {
+      return !unique.insert(yield).second;
+    }
+  };
+
+  // utility class to lazily create the k-best derivations from a forest, uses
+  // the lazy k-best algorithm (Algorithm 3) from Huang and Chiang (IWPT 2005)
+  template<typename T,  // yield type (returned by Traversal)
+           typename Traversal,
+           typename DerivationFilter = NoFilter,
+           typename WeightType = prob_t,
+           typename WeightFunction = EdgeProb>
+  struct KBestDerivations {
+    KBestDerivations(const Hypergraph& hg,
+                     const size_t k,
+                     const Traversal& tf = Traversal(),
+                     const WeightFunction& wf = WeightFunction()) :
+      traverse(tf), w(wf), g(hg), nds(g.nodes_.size()), k_prime(k) {}
+
+    ~KBestDerivations() {
+      for (int i = 0; i < freelist.size(); ++i)
+        delete freelist[i];
+    }
+
+    struct Derivation {
+      Derivation(const Hypergraph::Edge& e,
+                 const SmallVector& jv,
+                 const WeightType& w,
+                 const SparseVector<double>& f) :
+        edge(&e),
+        j(jv),
+        score(w),
+        feature_values(f) {}
+
+      // dummy constructor, just for query
+      Derivation(const Hypergraph::Edge& e,
+                 const SmallVector& jv) : edge(&e), j(jv) {}
+
+      T yield;
+      const Hypergraph::Edge* const edge;
+      const SmallVector j;
+      const WeightType score;
+      const SparseVector<double> feature_values;
+    };
+    struct HeapCompare {
+      bool operator()(const Derivation* a, const Derivation* b) const {
+        return a->score < b->score;
+      }
+    };
+    struct DerivationCompare {
+      bool operator()(const Derivation* a, const Derivation* b) const {
+        return a->score > b->score;
+      }
+    };
+    struct DerivationUniquenessHash {
+      size_t operator()(const Derivation* d) const {
+        size_t x = 5381;
+        x = ((x << 5) + x) ^ d->edge->id_;
+        for (int i = 0; i < d->j.size(); ++i)
+          x = ((x << 5) + x) ^ d->j[i];
+        return x;
+      }
+    };
+    struct DerivationUniquenessEquals {
+      bool operator()(const Derivation* a, const Derivation* b) const {
+        return (a->edge == b->edge) && (a->j == b->j);
+      }
+    };
+    typedef std::vector<Derivation*> CandidateHeap;
+    typedef std::vector<Derivation*> DerivationList;
+    typedef std::tr1::unordered_set<
+       const Derivation*, DerivationUniquenessHash, DerivationUniquenessEquals> UniqueDerivationSet;
+
+    struct NodeDerivationState {
+      CandidateHeap cand;
+      DerivationList D;
+      DerivationFilter filter;
+      UniqueDerivationSet ds;
+      explicit NodeDerivationState(const DerivationFilter& f = DerivationFilter()) : filter(f) {}
+    };
+
+    Derivation* LazyKthBest(int v, int k) {
+      NodeDerivationState& s = GetCandidates(v);
+      CandidateHeap& cand = s.cand;
+      DerivationList& D = s.D;
+      DerivationFilter& filter = s.filter;
+      bool add_next = true;
+      while (D.size() <= k) {
+        if (add_next && D.size() > 0) {
+          const Derivation* d = D.back();
+          LazyNext(d, &cand, &s.ds);
+        }
+        add_next = false;
+
+        if (cand.size() > 0) {
+          std::pop_heap(cand.begin(), cand.end(), HeapCompare());
+          Derivation* d = cand.back();
+          cand.pop_back();
+          std::vector<const T*> ants(d->edge->Arity());
+          for (int j = 0; j < ants.size(); ++j)
+            ants[j] = &LazyKthBest(d->edge->tail_nodes_[j], d->j[j])->yield;
+          traverse(*d->edge, ants, &d->yield);
+          if (!filter(d->yield)) {
+            D.push_back(d);
+            add_next = true;
+          }
+        } else {
+          break;
+        }
+      }
+      if (k < D.size()) return D[k]; else return NULL;
+    }
+
+  private:
+    // creates a derivation object with all fields set but the yield
+    // the yield is computed in LazyKthBest before the derivation is added to D
+    // returns NULL if j refers to derivation numbers larger than the
+    // antecedent structure define
+    Derivation* CreateDerivation(const Hypergraph::Edge& e, const SmallVector& j) {
+      WeightType score = w(e);
+      SparseVector<double> feats = e.feature_values_;
+      for (int i = 0; i < e.Arity(); ++i) {
+        const Derivation* ant = LazyKthBest(e.tail_nodes_[i], j[i]);
+        if (!ant) { return NULL; }
+        score *= ant->score;
+        feats += ant->feature_values;
+      }
+      freelist.push_back(new Derivation(e, j, score, feats));
+      return freelist.back();
+    }
+
+    NodeDerivationState& GetCandidates(int v) {
+      NodeDerivationState& s = nds[v];
+      if (!s.D.empty() || !s.cand.empty()) return s;
+
+      const Hypergraph::Node& node = g.nodes_[v];
+      for (int i = 0; i < node.in_edges_.size(); ++i) {
+        const Hypergraph::Edge& edge = g.edges_[node.in_edges_[i]];
+        SmallVector jv(edge.Arity(), 0);
+        Derivation* d = CreateDerivation(edge, jv);
+        assert(d);
+        s.cand.push_back(d);
+      }
+
+      const int effective_k = std::min(k_prime, s.cand.size());
+      const typename CandidateHeap::iterator kth = s.cand.begin() + effective_k;
+      std::nth_element(s.cand.begin(), kth, s.cand.end(), DerivationCompare());
+      s.cand.resize(effective_k);
+      std::make_heap(s.cand.begin(), s.cand.end(), HeapCompare());
+
+      return s;
+    }
+
+    void LazyNext(const Derivation* d, CandidateHeap* cand, UniqueDerivationSet* ds) {
+      for (int i = 0; i < d->j.size(); ++i) {
+        SmallVector j = d->j;
+        ++j[i];
+        const Derivation* ant = LazyKthBest(d->edge->tail_nodes_[i], j[i]);
+        if (ant) {
+          Derivation query_unique(*d->edge, j);
+          if (ds->count(&query_unique) == 0) {
+            Derivation* new_d = CreateDerivation(*d->edge, j);
+            if (new_d) {
+              cand->push_back(new_d);
+              std::push_heap(cand->begin(), cand->end(), HeapCompare());
+              bool inserted = ds->insert(new_d).second;  // insert into uniqueness set
+              assert(inserted);
+            }
+          }
+        }
+      }
+    }
+
+    const Traversal traverse;
+    const WeightFunction w;
+    const Hypergraph& g;
+    std::vector<NodeDerivationState> nds;
+    std::vector<Derivation*> freelist;
+    const size_t k_prime;
+  };
+}
+
+#endif
diff --git a/decoder/lattice.cc b/decoder/lattice.cc
new file mode 100644
index 00000000..e3631e59
--- /dev/null
+++ b/decoder/lattice.cc
@@ -0,0 +1,62 @@
+#include "lattice.h"
+
+#include "tdict.h"
+#include "hg_io.h"
+
+using namespace std;
+
+static const int kUNREACHABLE = 99999999;
+
+void Lattice::ComputeDistances() {
+  const int n = this->size() + 1;
+  dist_.resize(n, n, kUNREACHABLE);
+  for (int i = 0; i < this->size(); ++i) {
+    const vector<LatticeArc>& alts = (*this)[i];
+    for (int j = 0; j < alts.size(); ++j)
+      dist_(i, i + alts[j].dist2next) = 1;
+  }
+  for (int k = 0; k < n; ++k) {
+    for (int i = 0; i < n; ++i) {
+      for (int j = 0; j < n; ++j) {
+        const int dp = dist_(i,k) + dist_(k,j);
+        if (dist_(i,j) > dp)
+          dist_(i,j) = dp;
+      }
+    }
+  }
+
+  for (int i = 0; i < n; ++i) {
+    int latest = kUNREACHABLE;
+    for (int j = n-1; j >= 0; --j) {
+      const int c = dist_(i,j);
+      if (c < kUNREACHABLE)
+        latest = c;
+      else
+        dist_(i,j) = latest;
+    }
+  }
+  // cerr << dist_ << endl;
+}
+
+bool LatticeTools::LooksLikePLF(const string &line) {
+  return (line.size() > 5) && (line.substr(0,4) == "((('");
+}
+
+void LatticeTools::ConvertTextToLattice(const string& text, Lattice* pl) {
+  Lattice& l = *pl;
+  vector<WordID> ids;
+  TD::ConvertSentence(text, &ids);
+  l.resize(ids.size());
+  for (int i = 0; i < l.size(); ++i)
+    l[i].push_back(LatticeArc(ids[i], 0.0, 1));
+  l.is_sentence_ = true;
+}
+
+void LatticeTools::ConvertTextOrPLF(const string& text_or_plf, Lattice* pl) {
+  if (LooksLikePLF(text_or_plf))
+    HypergraphIO::PLFtoLattice(text_or_plf, pl);
+  else
+    ConvertTextToLattice(text_or_plf, pl);
+  pl->ComputeDistances();
+}
+
diff --git a/decoder/lattice.h b/decoder/lattice.h
new file mode 100644
index 00000000..ad4ca50d
--- /dev/null
+++ b/decoder/lattice.h
@@ -0,0 +1,46 @@
+#ifndef __LATTICE_H_
+#define __LATTICE_H_
+
+#include <string>
+#include <vector>
+#include "wordid.h"
+#include "array2d.h"
+
+class Lattice;
+struct LatticeTools {
+  static bool LooksLikePLF(const std::string &line);
+  static void ConvertTextToLattice(const std::string& text, Lattice* pl);
+  static void ConvertTextOrPLF(const std::string& text_or_plf, Lattice* pl);
+};
+
+struct LatticeArc {
+  WordID label;
+  double cost;
+  int dist2next;
+  LatticeArc() : label(), cost(), dist2next() {}
+  LatticeArc(WordID w, double c, int i) : label(w), cost(c), dist2next(i) {}
+};
+
+class Lattice : public std::vector<std::vector<LatticeArc> > {
+  friend void LatticeTools::ConvertTextOrPLF(const std::string& text_or_plf, Lattice* pl);
+  friend void LatticeTools::ConvertTextToLattice(const std::string& text, Lattice* pl);
+ public:
+  Lattice() : is_sentence_(false) {}
+  explicit Lattice(size_t t, const std::vector<LatticeArc>& v = std::vector<LatticeArc>()) :
+   std::vector<std::vector<LatticeArc> >(t, v),
+   is_sentence_(false) {}
+  int Distance(int from, int to) const {
+    if (dist_.empty())
+      return (to - from);
+    return dist_(from, to);
+  }
+  // TODO this should actually be computed based on the contents
+  // of the lattice
+  bool IsSentence() const { return is_sentence_; }
+ private:
+  void ComputeDistances();
+  Array2D<int> dist_;
+  bool is_sentence_;
+};
+
+#endif
diff --git a/decoder/lexalign.cc b/decoder/lexalign.cc
new file mode 100644
index 00000000..6adb1892
--- /dev/null
+++ b/decoder/lexalign.cc
@@ -0,0 +1,129 @@
+#include "lexalign.h"
+
+#include <iostream>
+
+#include "filelib.h"
+#include "hg.h"
+#include "tdict.h"
+#include "grammar.h"
+#include "sentence_metadata.h"
+
+using namespace std;
+
+struct LexicalAlignImpl {
+  LexicalAlignImpl(const boost::program_options::variables_map& conf) :
+      use_null(conf.count("lexcrf_use_null") > 0),
+      kXCAT(TD::Convert("X")*-1),
+      kNULL(TD::Convert("<eps>")),
+      kBINARY(new TRule("[X] ||| [X,1] [X,2] ||| [1] [2]")),
+      kGOAL_RULE(new TRule("[Goal] ||| [X,1] ||| [1]")) {
+  }
+
+  void BuildTrellis(const Lattice& lattice, const SentenceMetadata& smeta, Hypergraph* forest) {
+    const int e_len = smeta.GetTargetLength();
+    assert(e_len > 0);
+    const Lattice& target = smeta.GetReference();
+    const int f_len = lattice.size();
+    // hack to tell the feature function system how big the sentence pair is
+    const int f_start = (use_null ? -1 : 0);
+    int prev_node_id = -1;
+    for (int i = 0; i < e_len; ++i) {  // for each word in the *target*
+      const WordID& e_i = target[i][0].label;
+      Hypergraph::Node* node = forest->AddNode(kXCAT);
+      const int new_node_id = node->id_;
+      int num_srcs = 0;
+      for (int j = f_start; j < f_len; ++j) { // for each word in the source
+        const WordID src_sym = (j < 0 ? kNULL : lattice[j][0].label);
+        const TRulePtr& rule = LexRule(src_sym, e_i);
+        if (rule) {
+          Hypergraph::Edge* edge = forest->AddEdge(rule, Hypergraph::TailNodeVector());
+          edge->i_ = j;
+          edge->j_ = j+1;
+          edge->prev_i_ = i;
+          edge->prev_j_ = i+1;
+          edge->feature_values_ += edge->rule_->GetFeatureValues();
+          ++num_srcs;
+          forest->ConnectEdgeToHeadNode(edge->id_, new_node_id);
+        } else {
+          cerr << TD::Convert(src_sym) << " does not translate to " << TD::Convert(e_i) << endl;
+        }
+      }
+      assert(num_srcs > 0);
+      if (prev_node_id >= 0) {
+        const int comb_node_id = forest->AddNode(kXCAT)->id_;
+        Hypergraph::TailNodeVector tail(2, prev_node_id);
+        tail[1] = new_node_id;
+        Hypergraph::Edge* edge = forest->AddEdge(kBINARY, tail);
+        forest->ConnectEdgeToHeadNode(edge->id_, comb_node_id);
+        prev_node_id = comb_node_id;
+      } else {
+        prev_node_id = new_node_id;
+      }
+    }
+    Hypergraph::TailNodeVector tail(1, forest->nodes_.size() - 1);
+    Hypergraph::Node* goal = forest->AddNode(TD::Convert("Goal")*-1);
+    Hypergraph::Edge* hg_edge = forest->AddEdge(kGOAL_RULE, tail);
+    forest->ConnectEdgeToHeadNode(hg_edge, goal);
+  }
+
+  inline int LexFeatureId(const WordID& f, const WordID& e) {
+    map<int, int>& e2fid = f2e2fid[f];
+    map<int, int>::iterator it = e2fid.find(e);
+    if (it != e2fid.end())
+      return it->second;
+    int& fid = e2fid[e];
+    if (f == 0) {
+      fid = FD::Convert("Lx:<eps>_" + FD::Escape(TD::Convert(e)));
+    } else {
+      fid = FD::Convert("Lx:" + FD::Escape(TD::Convert(f)) + "_" + FD::Escape(TD::Convert(e)));
+    }
+    return fid;
+  }
+
+  inline const TRulePtr& LexRule(const WordID& f, const WordID& e) {
+    const int fid = LexFeatureId(f, e);
+    if (!fid) { return kNULL_PTR; }
+    map<int, TRulePtr>& e2rule = f2e2rule[f];
+    map<int, TRulePtr>::iterator it = e2rule.find(e);
+    if (it != e2rule.end())
+      return it->second;
+    TRulePtr& tr = e2rule[e];
+    tr.reset(TRule::CreateLexicalRule(f, e));
+    tr->scores_.set_value(fid, 1.0);
+    return tr;
+  }
+
+ private:
+  const bool use_null;
+  const WordID kXCAT;
+  const WordID kNULL;
+  const TRulePtr kBINARY;
+  const TRulePtr kGOAL_RULE;
+  const TRulePtr kNULL_PTR;
+  map<int, map<int, TRulePtr> > f2e2rule;
+  map<int, map<int, int> > f2e2fid;
+  GrammarPtr grammar;
+};
+
+LexicalAlign::LexicalAlign(const boost::program_options::variables_map& conf) :
+  pimpl_(new LexicalAlignImpl(conf)) {}
+
+bool LexicalAlign::TranslateImpl(const string& input,
+                      SentenceMetadata* smeta,
+                      const vector<double>& weights,
+                      Hypergraph* forest) {
+  Lattice& lattice = smeta->src_lattice_;
+  LatticeTools::ConvertTextOrPLF(input, &lattice);
+  if (!lattice.IsSentence()) {
+    // lexical models make independence assumptions
+    // that don't work with lattices or conf nets
+    cerr << "LexicalTrans: cannot deal with lattice source input!\n";
+    abort();
+  }
+  smeta->SetSourceLength(lattice.size());
+  pimpl_->BuildTrellis(lattice, *smeta, forest);
+  forest->is_linear_chain_ = true;
+  forest->Reweight(weights);
+  return true;
+}
+
diff --git a/decoder/lexalign.h b/decoder/lexalign.h
new file mode 100644
index 00000000..7ba4fe64
--- /dev/null
+++ b/decoder/lexalign.h
@@ -0,0 +1,18 @@
+#ifndef _LEXALIGN_H_
+#define _LEXALIGN_H_
+
+#include "translator.h"
+#include "lattice.h"
+
+struct LexicalAlignImpl;
+struct LexicalAlign : public Translator {
+  LexicalAlign(const boost::program_options::variables_map& conf);
+  bool TranslateImpl(const std::string& input,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* forest);
+ private:
+  boost::shared_ptr<LexicalAlignImpl> pimpl_;
+};
+
+#endif
diff --git a/decoder/lextrans.cc b/decoder/lextrans.cc
new file mode 100644
index 00000000..3fcd1a7d
--- /dev/null
+++ b/decoder/lextrans.cc
@@ -0,0 +1,119 @@
+#include "lextrans.h"
+
+#include <iostream>
+
+#include "filelib.h"
+#include "hg.h"
+#include "tdict.h"
+#include "grammar.h"
+#include "sentence_metadata.h"
+
+using namespace std;
+
+struct LexicalTransImpl {
+  LexicalTransImpl(const boost::program_options::variables_map& conf) :
+      use_null(conf.count("lexcrf_use_null") > 0),
+      kXCAT(TD::Convert("X")*-1),
+      kNULL(TD::Convert("<eps>")),
+      kBINARY(new TRule("[X] ||| [X,1] [X,2] ||| [1] [2]")),
+      kGOAL_RULE(new TRule("[Goal] ||| [X,1] ||| [1]")) {
+    vector<string> gfiles = conf["grammar"].as<vector<string> >();
+    assert(gfiles.size() == 1);
+    ReadFile rf(gfiles.front());
+    TextGrammar *tg = new TextGrammar;
+    grammar.reset(tg);
+    istream* in = rf.stream();
+    int lc = 0;
+    bool flag = false;
+    while(*in) {
+      string line;
+      getline(*in, line);
+      if (line.empty()) continue;
+      ++lc;
+      TRulePtr r(TRule::CreateRulePhrasetable(line));
+      tg->AddRule(r);
+      if (lc %   50000 == 0) { cerr << '.'; flag = true; }
+      if (lc % 2000000 == 0) { cerr << " [" << lc << "]\n"; flag = false; }
+    }
+    if (flag) cerr << endl;
+    cerr << "Loaded " << lc << " rules\n";
+  }
+
+  void BuildTrellis(const Lattice& lattice, const SentenceMetadata& smeta, Hypergraph* forest) {
+    const int e_len = smeta.GetTargetLength();
+    assert(e_len > 0);
+    const int f_len = lattice.size();
+    // hack to tell the feature function system how big the sentence pair is
+    const int f_start = (use_null ? -1 : 0);
+    int prev_node_id = -1;
+    for (int i = 0; i < e_len; ++i) {  // for each word in the *target*
+      Hypergraph::Node* node = forest->AddNode(kXCAT);
+      const int new_node_id = node->id_;
+      for (int j = f_start; j < f_len; ++j) { // for each word in the source
+        const WordID src_sym = (j < 0 ? kNULL : lattice[j][0].label);
+        const GrammarIter* gi = grammar->GetRoot()->Extend(src_sym);
+        if (!gi) {
+          cerr << "No translations found for: " << TD::Convert(src_sym) << "\n";
+          abort();
+        }
+        const RuleBin* rb = gi->GetRules();
+        assert(rb);
+        for (int k = 0; k < rb->GetNumRules(); ++k) {
+          TRulePtr rule = rb->GetIthRule(k);
+          Hypergraph::Edge* edge = forest->AddEdge(rule, Hypergraph::TailNodeVector());
+          edge->i_ = j;
+          edge->j_ = j+1;
+          edge->prev_i_ = i;
+          edge->prev_j_ = i+1;
+          edge->feature_values_ += edge->rule_->GetFeatureValues();
+          forest->ConnectEdgeToHeadNode(edge->id_, new_node_id);
+        }
+      }
+      if (prev_node_id >= 0) {
+        const int comb_node_id = forest->AddNode(kXCAT)->id_;
+        Hypergraph::TailNodeVector tail(2, prev_node_id);
+        tail[1] = new_node_id;
+        Hypergraph::Edge* edge = forest->AddEdge(kBINARY, tail);
+        forest->ConnectEdgeToHeadNode(edge->id_, comb_node_id);
+        prev_node_id = comb_node_id;
+      } else {
+        prev_node_id = new_node_id;
+      }
+    }
+    Hypergraph::TailNodeVector tail(1, forest->nodes_.size() - 1);
+    Hypergraph::Node* goal = forest->AddNode(TD::Convert("Goal")*-1);
+    Hypergraph::Edge* hg_edge = forest->AddEdge(kGOAL_RULE, tail);
+    forest->ConnectEdgeToHeadNode(hg_edge, goal);
+  }
+
+ private:
+  const bool use_null;
+  const WordID kXCAT;
+  const WordID kNULL;
+  const TRulePtr kBINARY;
+  const TRulePtr kGOAL_RULE;
+  GrammarPtr grammar;
+};
+
+LexicalTrans::LexicalTrans(const boost::program_options::variables_map& conf) :
+  pimpl_(new LexicalTransImpl(conf)) {}
+
+bool LexicalTrans::TranslateImpl(const string& input,
+                      SentenceMetadata* smeta,
+                      const vector<double>& weights,
+                      Hypergraph* forest) {
+  Lattice& lattice = smeta->src_lattice_;
+  LatticeTools::ConvertTextOrPLF(input, &lattice);
+  if (!lattice.IsSentence()) {
+    // lexical models make independence assumptions
+    // that don't work with lattices or conf nets
+    cerr << "LexicalTrans: cannot deal with lattice source input!\n";
+    abort();
+  }
+  smeta->SetSourceLength(lattice.size());
+  pimpl_->BuildTrellis(lattice, *smeta, forest);
+  forest->is_linear_chain_ = true;
+  forest->Reweight(weights);
+  return true;
+}
+
diff --git a/decoder/lextrans.h b/decoder/lextrans.h
new file mode 100644
index 00000000..2d51e7c0
--- /dev/null
+++ b/decoder/lextrans.h
@@ -0,0 +1,18 @@
+#ifndef _LEXTrans_H_
+#define _LEXTrans_H_
+
+#include "translator.h"
+#include "lattice.h"
+
+struct LexicalTransImpl;
+struct LexicalTrans : public Translator {
+  LexicalTrans(const boost::program_options::variables_map& conf);
+  bool TranslateImpl(const std::string& input,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* forest);
+ private:
+  boost::shared_ptr<LexicalTransImpl> pimpl_;
+};
+
+#endif
diff --git a/decoder/logval.h b/decoder/logval.h
new file mode 100644
index 00000000..7099b9be
--- /dev/null
+++ b/decoder/logval.h
@@ -0,0 +1,157 @@
+#ifndef LOGVAL_H_
+#define LOGVAL_H_
+
+#include <iostream>
+#include <cstdlib>
+#include <cmath>
+#include <limits>
+
+template <typename T>
+class LogVal {
+ public:
+  LogVal() : s_(), v_(-std::numeric_limits<T>::infinity()) {}
+  explicit LogVal(double x) : s_(std::signbit(x)), v_(s_ ? std::log(-x) : std::log(x)) {}
+  static LogVal<T> One() { return LogVal(1); }
+  static LogVal<T> Zero() { return LogVal(); }
+
+  void logeq(const T& v) { s_ = false; v_ = v; }
+
+  LogVal& operator+=(const LogVal& a) {
+    if (a.v_ == -std::numeric_limits<T>::infinity()) return *this;
+    if (a.s_ == s_) {
+      if (a.v_ < v_) {
+        v_ = v_ + log1p(std::exp(a.v_ - v_));
+      } else {
+        v_ = a.v_ + log1p(std::exp(v_ - a.v_));
+      }
+    } else {
+      if (a.v_ < v_) {
+        v_ = v_ + log1p(-std::exp(a.v_ - v_));
+      } else {
+        v_ = a.v_ + log1p(-std::exp(v_ - a.v_));
+        s_ = !s_;
+      }
+    }
+    return *this;
+  }
+
+  LogVal& operator*=(const LogVal& a) {
+    s_ = (s_ != a.s_);
+    v_ += a.v_;
+    return *this;
+  }
+
+  LogVal& operator/=(const LogVal& a) {
+    s_ = (s_ != a.s_);
+    v_ -= a.v_;
+    return *this;
+  }
+
+  LogVal& operator-=(const LogVal& a) {
+    LogVal b = a;
+    b.invert();
+    return *this += b;
+  }
+
+  LogVal& poweq(const T& power) {
+    if (s_) {
+      std::cerr << "poweq(T) not implemented when s_ is true\n";
+      std::abort();
+    } else {
+      v_ *= power;
+    }
+    return *this;
+  }
+
+  void invert() { s_ = !s_; }
+
+  LogVal pow(const T& power) const {
+    LogVal res = *this;
+    res.poweq(power);
+    return res;
+  }
+
+  operator T() const {
+    if (s_) return -std::exp(v_); else return std::exp(v_);
+  }
+
+  bool s_;
+  T v_;
+};
+
+template<typename T>
+LogVal<T> operator+(const LogVal<T>& o1, const LogVal<T>& o2) {
+  LogVal<T> res(o1);
+  res += o2;
+  return res;
+}
+
+template<typename T>
+LogVal<T> operator*(const LogVal<T>& o1, const LogVal<T>& o2) {
+  LogVal<T> res(o1);
+  res *= o2;
+  return res;
+}
+
+template<typename T>
+LogVal<T> operator/(const LogVal<T>& o1, const LogVal<T>& o2) {
+  LogVal<T> res(o1);
+  res /= o2;
+  return res;
+}
+
+template<typename T>
+LogVal<T> operator-(const LogVal<T>& o1, const LogVal<T>& o2) {
+  LogVal<T> res(o1);
+  res -= o2;
+  return res;
+}
+
+template<typename T>
+T log(const LogVal<T>& o) {
+  if (o.s_) return log(-1.0);
+  return o.v_;
+}
+
+template <typename T>
+LogVal<T> pow(const LogVal<T>& b, const T& e) {
+  return b.pow(e);
+}
+
+template <typename T>
+bool operator<(const LogVal<T>& lhs, const LogVal<T>& rhs) {
+  if (lhs.s_ == rhs.s_) {
+    return (lhs.v_ < rhs.v_);
+  } else {
+    return lhs.s_ > rhs.s_;
+  }
+}
+
+#if 0
+template <typename T>
+bool operator<=(const LogVal<T>& lhs, const LogVal<T>& rhs) {
+  return (lhs.v_ <= rhs.v_);
+}
+
+template <typename T>
+bool operator>(const LogVal<T>& lhs, const LogVal<T>& rhs) {
+  return (lhs.v_ > rhs.v_);
+}
+
+template <typename T>
+bool operator>=(const LogVal<T>& lhs, const LogVal<T>& rhs) {
+  return (lhs.v_ >= rhs.v_);
+}
+#endif
+
+template <typename T>
+bool operator==(const LogVal<T>& lhs, const LogVal<T>& rhs) {
+  return (lhs.v_ == rhs.v_) && (lhs.s_ == rhs.s_);
+}
+
+template <typename T>
+bool operator!=(const LogVal<T>& lhs, const LogVal<T>& rhs) {
+  return !(lhs == rhs);
+}
+
+#endif
diff --git a/decoder/logval_test.cc b/decoder/logval_test.cc
new file mode 100644
index 00000000..1a23177d
--- /dev/null
+++ b/decoder/logval_test.cc
@@ -0,0 +1,73 @@
+#include "logval.h"
+
+#include <gtest/gtest.h>
+#include <iostream>
+
+class LogValTest : public testing::Test {
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+};
+
+using namespace std;
+
+TEST_F(LogValTest,Order) {
+  LogVal<double> a(-0.3);
+  LogVal<double> b(0.3);
+  LogVal<double> c(2.4);
+  EXPECT_LT(a,b);
+  EXPECT_LT(b,c);
+  EXPECT_LT(a,c);
+  EXPECT_FALSE(b < a);
+  EXPECT_FALSE(c < a);
+  EXPECT_FALSE(c < b);
+  EXPECT_FALSE(c < c);
+  EXPECT_FALSE(b < b);
+  EXPECT_FALSE(a < a);
+}
+
+TEST_F(LogValTest,Invert) {
+  LogVal<double> x(-2.4);
+  LogVal<double> y(2.4);
+  y.invert();
+  EXPECT_FLOAT_EQ(x,y);
+}
+
+TEST_F(LogValTest,Minus) {
+  LogVal<double> x(12);
+  LogVal<double> y(2);
+  LogVal<double> z1 = x - y;
+  LogVal<double> z2 = x;
+  z2 -= y;
+  EXPECT_FLOAT_EQ(z1, z2);
+  EXPECT_FLOAT_EQ(z1, 10.0);
+  EXPECT_FLOAT_EQ(y - x, -10.0);
+}
+
+TEST_F(LogValTest,TestOps) {
+  LogVal<double> x(-12.12);
+  LogVal<double> y(x);
+  cerr << x << endl;
+  cerr << (x*y) << endl;
+  cerr << (x*y + x) << endl;
+  cerr << (x + x*y) << endl;
+  cerr << log1p(-0.5) << endl;
+  LogVal<double> aa(0.2);
+  LogVal<double> bb(-0.3);
+  cerr << (aa + bb) << endl;
+  cerr << (bb + aa) << endl;
+  EXPECT_FLOAT_EQ((aa + bb), (bb + aa));
+  EXPECT_FLOAT_EQ((aa + bb), -0.1);
+}
+
+TEST_F(LogValTest,TestSizes) {
+  cerr << sizeof(LogVal<double>) << endl;
+  cerr << sizeof(LogVal<float>) << endl;
+  cerr << sizeof(void*) << endl;
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
diff --git a/decoder/maxtrans_blunsom.cc b/decoder/maxtrans_blunsom.cc
new file mode 100644
index 00000000..34e175db
--- /dev/null
+++ b/decoder/maxtrans_blunsom.cc
@@ -0,0 +1,287 @@
+#include "apply_models.h"
+
+#include <vector>
+#include <algorithm>
+#include <tr1/unordered_map>
+#include <tr1/unordered_set>
+
+#include <boost/tuple/tuple.hpp>
+#include <boost/functional/hash.hpp>
+
+#include "tdict.h"
+#include "hg.h"
+#include "ff.h"
+
+using boost::tuple;
+using namespace std;
+using namespace std::tr1;
+
+namespace Hack {
+
+struct Candidate;
+typedef SmallVector JVector;
+typedef vector<Candidate*> CandidateHeap;
+typedef vector<Candidate*> CandidateList;
+
+// life cycle: candidates are created, placed on the heap
+// and retrieved by their estimated cost, when they're
+// retrieved, they're incorporated into the +LM hypergraph
+// where they also know the head node index they are
+// attached to.  After they are added to the +LM hypergraph
+// inside_prob_ and est_prob_ fields may be updated as better
+// derivations are found (this happens since the successor's
+// of derivation d may have a better score- they are
+// explored lazily).  However, the updates don't happen
+// when a candidate is in the heap so maintaining the heap
+// property is not an issue.
+struct Candidate {
+  int node_index_;                     // -1 until incorporated
+                                       // into the +LM forest
+  const Hypergraph::Edge* in_edge_;    // in -LM forest
+  Hypergraph::Edge out_edge_;
+  vector<WordID> state_;
+  const JVector j_;
+  prob_t inside_prob_;            // these are fixed until the cand
+                               // is popped, then they may be updated
+  prob_t est_prob_;
+
+  Candidate(const Hypergraph::Edge& e,
+            const JVector& j,
+            const vector<CandidateList>& D,
+            bool is_goal) :
+      node_index_(-1),
+      in_edge_(&e),
+      j_(j) {
+    InitializeCandidate(D, is_goal);
+  }
+
+  // used to query uniqueness
+  Candidate(const Hypergraph::Edge& e,
+            const JVector& j) : in_edge_(&e), j_(j) {}
+
+  bool IsIncorporatedIntoHypergraph() const {
+    return node_index_ >= 0;
+  }
+
+  void InitializeCandidate(const vector<vector<Candidate*> >& D,
+                           const bool is_goal) {
+    const Hypergraph::Edge& in_edge = *in_edge_;
+    out_edge_.rule_ = in_edge.rule_;
+    out_edge_.feature_values_ = in_edge.feature_values_;
+    Hypergraph::TailNodeVector& tail = out_edge_.tail_nodes_;
+    tail.resize(j_.size());
+    prob_t p = prob_t::One();
+    // cerr << "\nEstimating application of " << in_edge.rule_->AsString() << endl;
+    vector<const vector<WordID>* > ants(tail.size());
+    for (int i = 0; i < tail.size(); ++i) {
+      const Candidate& ant = *D[in_edge.tail_nodes_[i]][j_[i]];
+      ants[i] = &ant.state_;
+      assert(ant.IsIncorporatedIntoHypergraph());
+      tail[i] = ant.node_index_;
+      p *= ant.inside_prob_;
+    }
+    prob_t edge_estimate = prob_t::One();
+    if (is_goal) {
+      assert(tail.size() == 1);
+      out_edge_.edge_prob_ = in_edge.edge_prob_;
+    } else {
+      in_edge.rule_->ESubstitute(ants, &state_);
+      out_edge_.edge_prob_ = in_edge.edge_prob_;
+    }
+    inside_prob_ = out_edge_.edge_prob_ * p;
+    est_prob_ = inside_prob_ * edge_estimate;
+  }
+};
+
+ostream& operator<<(ostream& os, const Candidate& cand) {
+  os << "CAND[";
+  if (!cand.IsIncorporatedIntoHypergraph()) { os << "PENDING "; }
+  else { os << "+LM_node=" << cand.node_index_; }
+  os << " edge=" << cand.in_edge_->id_;
+  os << " j=<";
+  for (int i = 0; i < cand.j_.size(); ++i)
+    os << (i==0 ? "" : " ") << cand.j_[i];
+  os << "> vit=" << log(cand.inside_prob_);
+  os << " est=" << log(cand.est_prob_);
+  return os << ']';
+}
+
+struct HeapCandCompare {
+  bool operator()(const Candidate* l, const Candidate* r) const {
+    return l->est_prob_ < r->est_prob_;
+  }
+};
+
+struct EstProbSorter {
+  bool operator()(const Candidate* l, const Candidate* r) const {
+    return l->est_prob_ > r->est_prob_;
+  }
+};
+
+// the same candidate <edge, j> can be added multiple times if
+// j is multidimensional (if you're going NW in Manhattan, you
+// can first go north, then west, or you can go west then north)
+// this is a hash function on the relevant variables from
+// Candidate to enforce this.
+struct CandidateUniquenessHash {
+  size_t operator()(const Candidate* c) const {
+    size_t x = 5381;
+    x = ((x << 5) + x) ^ c->in_edge_->id_;
+    for (int i = 0; i < c->j_.size(); ++i)
+      x = ((x << 5) + x) ^ c->j_[i];
+    return x;
+  }
+};
+
+struct CandidateUniquenessEquals {
+  bool operator()(const Candidate* a, const Candidate* b) const {
+    return (a->in_edge_ == b->in_edge_) && (a->j_ == b->j_);
+  }
+};
+
+typedef unordered_set<const Candidate*, CandidateUniquenessHash, CandidateUniquenessEquals> UniqueCandidateSet;
+typedef unordered_map<vector<WordID>, Candidate*, boost::hash<vector<WordID> > > State2Node;
+
+class MaxTransBeamSearch {
+
+public:
+  MaxTransBeamSearch(const Hypergraph& i, int pop_limit, Hypergraph* o) :
+      in(i),
+      out(*o),
+      D(in.nodes_.size()),
+      pop_limit_(pop_limit) {
+    cerr << "  Finding max translation (cube pruning, pop_limit = " << pop_limit_ << ')' << endl;
+  }
+
+  void Apply() {
+    int num_nodes = in.nodes_.size();
+    int goal_id = num_nodes - 1;
+    int pregoal = goal_id - 1;
+    assert(in.nodes_[pregoal].out_edges_.size() == 1);
+    cerr << "    ";
+    for (int i = 0; i < in.nodes_.size(); ++i) {
+      cerr << '.';
+      KBest(i, i == goal_id);
+    }
+    cerr << endl;
+    int best_node = D[goal_id].front()->in_edge_->tail_nodes_.front();
+    Candidate& best = *D[best_node].front();
+    cerr << "  Best path: " << log(best.inside_prob_)
+         << "\t" << log(best.est_prob_) << endl;
+    cout << TD::GetString(D[best_node].front()->state_) << endl;
+    FreeAll();
+  }
+
+ private:
+  void FreeAll() {
+    for (int i = 0; i < D.size(); ++i) {
+      CandidateList& D_i = D[i];
+      for (int j = 0; j < D_i.size(); ++j)
+        delete D_i[j];
+    }
+    D.clear();
+  }
+
+  void IncorporateIntoPlusLMForest(Candidate* item, State2Node* s2n, CandidateList* freelist) {
+    Hypergraph::Edge* new_edge = out.AddEdge(item->out_edge_.rule_, item->out_edge_.tail_nodes_);
+    new_edge->feature_values_ = item->out_edge_.feature_values_;
+    new_edge->edge_prob_ = item->out_edge_.edge_prob_;
+    Candidate*& o_item = (*s2n)[item->state_];
+    if (!o_item) o_item = item;
+    
+    int& node_id = o_item->node_index_;
+    if (node_id < 0) {
+      Hypergraph::Node* new_node = out.AddNode(in.nodes_[item->in_edge_->head_node_].cat_);
+      node_id = new_node->id_;
+    }
+    Hypergraph::Node* node = &out.nodes_[node_id];
+    out.ConnectEdgeToHeadNode(new_edge, node);
+
+    if (item != o_item) {
+      assert(o_item->state_ == item->state_);    // sanity check!
+      o_item->est_prob_ += item->est_prob_;
+      o_item->inside_prob_ += item->inside_prob_;
+      freelist->push_back(item);
+    }
+  }
+
+  void KBest(const int vert_index, const bool is_goal) {
+    // cerr << "KBest(" << vert_index << ")\n";
+    CandidateList& D_v = D[vert_index];
+    assert(D_v.empty());
+    const Hypergraph::Node& v = in.nodes_[vert_index];
+    // cerr << "  has " << v.in_edges_.size() << " in-coming edges\n";
+    const vector<int>& in_edges = v.in_edges_;
+    CandidateHeap cand;
+    CandidateList freelist;
+    cand.reserve(in_edges.size());
+    UniqueCandidateSet unique_cands;
+    for (int i = 0; i < in_edges.size(); ++i) {
+      const Hypergraph::Edge& edge = in.edges_[in_edges[i]];
+      const JVector j(edge.tail_nodes_.size(), 0);
+      cand.push_back(new Candidate(edge, j, D, is_goal));
+      assert(unique_cands.insert(cand.back()).second);  // these should all be unique!
+    }
+//    cerr << "  making heap of " << cand.size() << " candidates\n";
+    make_heap(cand.begin(), cand.end(), HeapCandCompare());
+    State2Node state2node;   // "buf" in Figure 2
+    int pops = 0;
+    while(!cand.empty() && pops < pop_limit_) {
+      pop_heap(cand.begin(), cand.end(), HeapCandCompare());
+      Candidate* item = cand.back();
+      cand.pop_back();
+      // cerr << "POPPED: " << *item << endl;
+      PushSucc(*item, is_goal, &cand, &unique_cands);
+      IncorporateIntoPlusLMForest(item, &state2node, &freelist);
+      ++pops;
+    }
+    D_v.resize(state2node.size());
+    int c = 0;
+    for (State2Node::iterator i = state2node.begin(); i != state2node.end(); ++i)
+      D_v[c++] = i->second;
+    sort(D_v.begin(), D_v.end(), EstProbSorter());
+    // cerr << "  expanded to " << D_v.size() << " nodes\n";
+
+    for (int i = 0; i < cand.size(); ++i)
+      delete cand[i];
+    // freelist is necessary since even after an item merged, it still stays in
+    // the unique set so it can't be deleted til now
+    for (int i = 0; i < freelist.size(); ++i)
+      delete freelist[i];
+  }
+
+  void PushSucc(const Candidate& item, const bool is_goal, CandidateHeap* pcand, UniqueCandidateSet* cs) {
+    CandidateHeap& cand = *pcand;
+    for (int i = 0; i < item.j_.size(); ++i) {
+      JVector j = item.j_;
+      ++j[i];
+      if (j[i] < D[item.in_edge_->tail_nodes_[i]].size()) {
+        Candidate query_unique(*item.in_edge_, j);
+        if (cs->count(&query_unique) == 0) {
+          Candidate* new_cand = new Candidate(*item.in_edge_, j, D, is_goal);
+          cand.push_back(new_cand);
+          push_heap(cand.begin(), cand.end(), HeapCandCompare());
+          assert(cs->insert(new_cand).second);  // insert into uniqueness set, sanity check
+        }
+      }
+    }
+  }
+
+  const Hypergraph& in;
+  Hypergraph& out;
+
+  vector<CandidateList> D;   // maps nodes in in-HG to the
+                                   // equivalent nodes (many due to state
+                                   // splits) in the out-HG.
+  const int pop_limit_;
+};
+
+// each node in the graph has one of these, it keeps track of
+void MaxTrans(const Hypergraph& in,
+              int beam_size) {
+  Hypergraph out;
+  MaxTransBeamSearch ma(in, beam_size, &out);
+  ma.Apply();
+}
+
+}
diff --git a/decoder/parser_test.cc b/decoder/parser_test.cc
new file mode 100644
index 00000000..da1fbd89
--- /dev/null
+++ b/decoder/parser_test.cc
@@ -0,0 +1,35 @@
+#include <cassert>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <gtest/gtest.h>
+#include "hg.h"
+#include "trule.h"
+#include "bottom_up_parser.h"
+#include "tdict.h"
+
+using namespace std;
+
+class ChartTest : public testing::Test {
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+};
+       
+TEST_F(ChartTest,LanguageModel) {
+  LatticeArc a(TD::Convert("ein"), 0.0, 1);
+  LatticeArc b(TD::Convert("haus"), 0.0, 1);
+  Lattice lattice(2);
+  lattice[0].push_back(a);
+  lattice[1].push_back(b);
+  Hypergraph forest;
+  GrammarPtr g(new TextGrammar);
+  vector<GrammarPtr> grammars(1, g);
+  ExhaustiveBottomUpParser parser("PHRASE", grammars);
+  parser.Parse(lattice, &forest);
+}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/decoder/phrasebased_translator.cc b/decoder/phrasebased_translator.cc
new file mode 100644
index 00000000..726b3f9a
--- /dev/null
+++ b/decoder/phrasebased_translator.cc
@@ -0,0 +1,206 @@
+#include "phrasebased_translator.h"
+
+#include <queue>
+#include <iostream>
+#include <tr1/unordered_map>
+#include <tr1/unordered_set>
+
+#include <boost/tuple/tuple.hpp>
+#include <boost/functional/hash.hpp>
+
+#include "sentence_metadata.h"
+#include "tdict.h"
+#include "hg.h"
+#include "filelib.h"
+#include "lattice.h"
+#include "phrasetable_fst.h"
+#include "array2d.h"
+
+using namespace std;
+using namespace std::tr1;
+using namespace boost::tuples;
+
+struct Coverage : public vector<bool> {
+  explicit Coverage(int n, bool v = false) : vector<bool>(n, v), first_gap() {}
+  void Cover(int i, int j) {
+    vector<bool>::iterator it = this->begin() + i;
+    vector<bool>::iterator end = this->begin() + j;
+    while (it != end)
+      *it++ = true;
+    if (first_gap == i) {
+      first_gap = j;
+      it = end;
+      while (*it && it != this->end()) {
+        ++it;
+        ++first_gap;
+      }
+    }
+  }
+  bool Collides(int i, int j) const {
+    vector<bool>::const_iterator it = this->begin() + i;
+    vector<bool>::const_iterator end = this->begin() + j;
+    while (it != end)
+      if (*it++) return true;
+    return false;
+  }
+  int GetFirstGap() const { return first_gap; }
+ private:
+  int first_gap;
+};
+struct CoverageHash {
+  size_t operator()(const Coverage& cov) const {
+    return hasher_(static_cast<const vector<bool>&>(cov));
+  }
+ private:
+  boost::hash<vector<bool> > hasher_;
+};
+ostream& operator<<(ostream& os, const Coverage& cov) {
+  os << '[';
+  for (int i = 0; i < cov.size(); ++i)
+    os << (cov[i] ? '*' : '.');
+  return os << " gap=" << cov.GetFirstGap() << ']';
+}
+
+typedef unordered_map<Coverage, int, CoverageHash> CoverageNodeMap;
+typedef unordered_set<Coverage, CoverageHash> UniqueCoverageSet;
+
+struct PhraseBasedTranslatorImpl {
+  PhraseBasedTranslatorImpl(const boost::program_options::variables_map& conf) :
+      add_pass_through_rules(conf.count("add_pass_through_rules")),
+      max_distortion(conf["pb_max_distortion"].as<int>()),
+      kSOURCE_RULE(new TRule("[X] ||| [X,1] ||| [X,1]", true)),
+      kCONCAT_RULE(new TRule("[X] ||| [X,1] [X,2] ||| [X,1] [X,2]", true)),
+      kNT_TYPE(TD::Convert("X") * -1) {
+    assert(max_distortion >= 0);
+    vector<string> gfiles = conf["grammar"].as<vector<string> >();
+    assert(gfiles.size() == 1);
+    cerr << "Reading phrasetable from " << gfiles.front() << endl;
+    ReadFile in(gfiles.front());
+    fst.reset(LoadTextPhrasetable(in.stream()));
+  }
+
+  struct State {
+    State(const Coverage& c, int _i, int _j, const FSTNode* q) :
+      coverage(c), i(_i), j(_j), fst(q) {}
+    Coverage coverage;
+    int i;
+    int j;
+    const FSTNode* fst;
+  };
+
+  // we keep track of unique coverages that have been extended since it's
+  // possible to "extend" the same coverage twice, e.g. translate "a b c"
+  // with phrases "a" "b" "a b" and "c".  There are two ways to cover "a b"
+  void EnqueuePossibleContinuations(const Coverage& coverage, queue<State>* q, UniqueCoverageSet* ucs) {
+    if (ucs->insert(coverage).second) {
+      const int gap = coverage.GetFirstGap();
+      const int end = min(static_cast<int>(coverage.size()), gap + max_distortion + 1);
+      for (int i = gap; i < end; ++i)
+        if (!coverage[i]) q->push(State(coverage, i, i, fst.get()));
+    }
+  }
+
+  bool Translate(const std::string& input,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* minus_lm_forest) {
+    Lattice lattice;
+    LatticeTools::ConvertTextOrPLF(input, &lattice);
+    smeta->SetSourceLength(lattice.size());
+    size_t est_nodes = lattice.size() * lattice.size() * (1 << max_distortion);
+    minus_lm_forest->ReserveNodes(est_nodes, est_nodes * 100);
+    if (add_pass_through_rules) {
+      SparseVector<double> feats;
+      feats.set_value(FD::Convert("PassThrough"), 1);
+      for (int i = 0; i < lattice.size(); ++i) {
+        const vector<LatticeArc>& arcs = lattice[i];
+        for (int j = 0; j < arcs.size(); ++j) {
+          fst->AddPassThroughTranslation(arcs[j].label, feats);
+          // TODO handle lattice edge features
+        }
+      }
+    }
+    CoverageNodeMap c;
+    queue<State> q;
+    UniqueCoverageSet ucs;
+    const Coverage empty_cov(lattice.size(), false);
+    const Coverage goal_cov(lattice.size(), true);
+    EnqueuePossibleContinuations(empty_cov, &q, &ucs);
+    c[empty_cov] = 0;   // have to handle the left edge specially
+    while(!q.empty()) {
+      const State s = q.front();
+      q.pop();
+      // cerr << "(" << s.i << "," << s.j << " ptr=" << s.fst << ") cov=" << s.coverage << endl;
+      const vector<LatticeArc>& arcs = lattice[s.j];
+      if (s.fst->HasData()) {
+        Coverage new_cov = s.coverage;
+        new_cov.Cover(s.i, s.j);
+        EnqueuePossibleContinuations(new_cov, &q, &ucs);
+        const vector<TRulePtr>& phrases = s.fst->GetTranslations()->GetRules();
+        const int phrase_head_index = minus_lm_forest->AddNode(kNT_TYPE)->id_;
+        for (int i = 0; i < phrases.size(); ++i) {
+          Hypergraph::Edge* edge = minus_lm_forest->AddEdge(phrases[i], Hypergraph::TailNodeVector());
+          edge->feature_values_ = edge->rule_->scores_;
+          minus_lm_forest->ConnectEdgeToHeadNode(edge->id_, phrase_head_index);
+        }
+        CoverageNodeMap::iterator cit = c.find(s.coverage);
+        assert(cit != c.end());
+        const int tail_node_plus1 = cit->second;
+        if (tail_node_plus1 == 0) {  // left edge
+          c[new_cov] = phrase_head_index + 1;
+        } else { // not left edge
+          int& head_node_plus1 = c[new_cov];
+          if (!head_node_plus1)
+            head_node_plus1 = minus_lm_forest->AddNode(kNT_TYPE)->id_ + 1;
+          Hypergraph::TailNodeVector tail(2, tail_node_plus1 - 1);
+          tail[1] = phrase_head_index;
+          const int concat_edge = minus_lm_forest->AddEdge(kCONCAT_RULE, tail)->id_;
+          minus_lm_forest->ConnectEdgeToHeadNode(concat_edge, head_node_plus1 - 1);
+        }
+      }
+      if (s.j == lattice.size()) continue;
+      for (int l = 0; l < arcs.size(); ++l) {
+        const LatticeArc& arc = arcs[l];
+
+        const FSTNode* next_fst_state = s.fst->Extend(arc.label);
+        const int next_j = s.j + arc.dist2next;
+        if (next_fst_state &&
+            !s.coverage.Collides(s.i, next_j)) {
+          q.push(State(s.coverage, s.i, next_j, next_fst_state));
+        }
+      }
+    }
+    if (add_pass_through_rules)
+      fst->ClearPassThroughTranslations();
+    int pregoal_plus1 = c[goal_cov];
+    if (pregoal_plus1 > 0) {
+      TRulePtr kGOAL_RULE(new TRule("[Goal] ||| [X,1] ||| [X,1]"));
+      int goal = minus_lm_forest->AddNode(TD::Convert("Goal") * -1)->id_;
+      int gedge = minus_lm_forest->AddEdge(kGOAL_RULE, Hypergraph::TailNodeVector(1, pregoal_plus1 - 1))->id_;
+      minus_lm_forest->ConnectEdgeToHeadNode(gedge, goal);
+      // they are almost topo, but not quite always
+      minus_lm_forest->TopologicallySortNodesAndEdges(goal);
+      minus_lm_forest->Reweight(weights);
+      return true;
+    } else {
+      return false;  // composition failed
+    }
+  }
+
+  const bool add_pass_through_rules;
+  const int max_distortion;
+  TRulePtr kSOURCE_RULE;
+  const TRulePtr kCONCAT_RULE;
+  const WordID kNT_TYPE;
+  boost::shared_ptr<FSTNode> fst;
+};
+
+PhraseBasedTranslator::PhraseBasedTranslator(const boost::program_options::variables_map& conf) :
+  pimpl_(new PhraseBasedTranslatorImpl(conf)) {}
+
+bool PhraseBasedTranslator::TranslateImpl(const std::string& input,
+                                      SentenceMetadata* smeta,
+                                      const std::vector<double>& weights,
+                                      Hypergraph* minus_lm_forest) {
+  return pimpl_->Translate(input, smeta, weights, minus_lm_forest);
+}
diff --git a/decoder/phrasebased_translator.h b/decoder/phrasebased_translator.h
new file mode 100644
index 00000000..e5e3f8a2
--- /dev/null
+++ b/decoder/phrasebased_translator.h
@@ -0,0 +1,18 @@
+#ifndef _PHRASEBASED_TRANSLATOR_H_
+#define _PHRASEBASED_TRANSLATOR_H_
+
+#include "translator.h"
+
+class PhraseBasedTranslatorImpl;
+class PhraseBasedTranslator : public Translator {
+ public:
+  PhraseBasedTranslator(const boost::program_options::variables_map& conf);
+  bool TranslateImpl(const std::string& input,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* minus_lm_forest);
+ private:
+  boost::shared_ptr<PhraseBasedTranslatorImpl> pimpl_;
+};
+
+#endif
diff --git a/decoder/phrasetable_fst.cc b/decoder/phrasetable_fst.cc
new file mode 100644
index 00000000..f421e941
--- /dev/null
+++ b/decoder/phrasetable_fst.cc
@@ -0,0 +1,141 @@
+#include "phrasetable_fst.h"
+
+#include <cassert>
+#include <iostream>
+#include <map>
+
+#include <boost/shared_ptr.hpp>
+
+#include "filelib.h"
+#include "tdict.h"
+
+using boost::shared_ptr;
+using namespace std;
+
+TargetPhraseSet::~TargetPhraseSet() {}
+FSTNode::~FSTNode() {}
+
+class TextTargetPhraseSet : public TargetPhraseSet {
+ public:
+  void AddRule(TRulePtr rule) {
+    rules_.push_back(rule);
+  }
+  const vector<TRulePtr>& GetRules() const {
+    return rules_;
+  }
+
+ private:
+  // all rules must have arity 0
+  vector<TRulePtr> rules_;
+};
+
+class TextFSTNode : public FSTNode {
+ public:
+  const TargetPhraseSet* GetTranslations() const { return data.get(); }
+  bool HasData() const { return (bool)data; }
+  bool HasOutgoingNonEpsilonEdges() const { return !ptr.empty(); } 
+  const FSTNode* Extend(const WordID& t) const {
+    map<WordID, TextFSTNode>::const_iterator it = ptr.find(t);
+    if (it == ptr.end()) return NULL;
+    return &it->second;
+  }
+
+  void AddPhrase(const string& phrase);
+
+  void AddPassThroughTranslation(const WordID& w, const SparseVector<double>& feats);
+  void ClearPassThroughTranslations();
+ private:
+  vector<WordID> passthroughs;
+  shared_ptr<TargetPhraseSet> data;
+  map<WordID, TextFSTNode> ptr;
+};
+
+#ifdef DEBUG_CHART_PARSER
+static string TrimRule(const string& r) {
+  size_t start = r.find(" |||") + 5;
+  size_t end = r.rfind(" |||");
+  return r.substr(start, end - start);
+}
+#endif
+
+void TextFSTNode::AddPhrase(const string& phrase) {
+  vector<WordID> words;
+  TRulePtr rule(TRule::CreateRulePhrasetable(phrase));
+  if (!rule) {
+    static int err = 0;
+    ++err;
+    if (err > 2) { cerr << "TOO MANY PHRASETABLE ERRORS\n"; exit(1); }
+    return;
+  }
+
+  TextFSTNode* fsa = this;
+  for (int i = 0; i < rule->FLength(); ++i)
+    fsa = &fsa->ptr[rule->f_[i]];
+
+  if (!fsa->data)
+    fsa->data.reset(new TextTargetPhraseSet);
+  static_cast<TextTargetPhraseSet*>(fsa->data.get())->AddRule(rule);
+}
+
+void TextFSTNode::AddPassThroughTranslation(const WordID& w, const SparseVector<double>& feats) {
+  TextFSTNode* next = &ptr[w];
+  // current, rules are only added if the symbol is completely missing as a
+  // word starting the phrase.  As a result, it is possible that some sentences
+  // won't parse.  If this becomes a problem, fix it here.
+  if (!next->data) {
+    TextTargetPhraseSet* tps = new TextTargetPhraseSet;
+    next->data.reset(tps);
+    TRule* rule = new TRule;
+    rule->e_.resize(1, w);
+    rule->f_.resize(1, w);
+    rule->lhs_ = TD::Convert("___PHRASE") * -1;
+    rule->scores_ = feats;
+    rule->arity_ = 0;
+    tps->AddRule(TRulePtr(rule));
+    passthroughs.push_back(w); 
+  }
+}
+
+void TextFSTNode::ClearPassThroughTranslations() {
+  for (int i = 0; i < passthroughs.size(); ++i)
+    ptr.erase(passthroughs[i]);
+  passthroughs.clear();
+}
+
+static void AddPhrasetableToFST(istream* in, TextFSTNode* fst) {
+  int lc = 0;
+  bool flag = false;
+  while(*in) {
+    string line;
+    getline(*in, line);
+    if (line.empty()) continue;
+    ++lc;
+    fst->AddPhrase(line);
+    if (lc % 10000 == 0) { flag = true; cerr << '.' << flush; }
+    if (lc % 500000 == 0) { flag = false; cerr << " [" << lc << ']' << endl << flush; }
+  }
+  if (flag) cerr << endl;
+  cerr << "Loaded " << lc << " source phrases\n";
+}
+
+FSTNode* LoadTextPhrasetable(istream* in) {
+  TextFSTNode *fst = new TextFSTNode;
+  AddPhrasetableToFST(in, fst);
+  return fst;
+}
+
+FSTNode* LoadTextPhrasetable(const vector<string>& filenames) {
+  TextFSTNode* fst = new TextFSTNode;
+  for (int i = 0; i < filenames.size(); ++i) {
+    ReadFile rf(filenames[i]);
+    cerr << "Reading phrase from " << filenames[i] << endl;
+    AddPhrasetableToFST(rf.stream(), fst);
+  }
+  return fst;
+}
+
+FSTNode* LoadBinaryPhrasetable(const string& fname_prefix) {
+  (void) fname_prefix;
+  assert(!"not implemented yet");
+}
+
diff --git a/decoder/phrasetable_fst.h b/decoder/phrasetable_fst.h
new file mode 100644
index 00000000..477de1f7
--- /dev/null
+++ b/decoder/phrasetable_fst.h
@@ -0,0 +1,34 @@
+#ifndef _PHRASETABLE_FST_H_
+#define _PHRASETABLE_FST_H_
+
+#include <vector>
+#include <string>
+
+#include "sparse_vector.h"
+#include "trule.h"
+
+class TargetPhraseSet {
+ public:
+  virtual ~TargetPhraseSet();
+  virtual const std::vector<TRulePtr>& GetRules() const = 0;
+};
+
+class FSTNode {
+ public:
+  virtual ~FSTNode();
+  virtual const TargetPhraseSet* GetTranslations() const = 0;
+  virtual bool HasData() const = 0;
+  virtual bool HasOutgoingNonEpsilonEdges() const = 0;
+  virtual const FSTNode* Extend(const WordID& t) const = 0;
+
+  // these should only be called on q_0:
+  virtual void AddPassThroughTranslation(const WordID& w, const SparseVector<double>& feats) = 0;
+  virtual void ClearPassThroughTranslations() = 0;
+};
+
+// attn caller: you own the memory
+FSTNode* LoadTextPhrasetable(const std::vector<std::string>& filenames);
+FSTNode* LoadTextPhrasetable(std::istream* in);
+FSTNode* LoadBinaryPhrasetable(const std::string& fname_prefix);
+
+#endif
diff --git a/decoder/prob.h b/decoder/prob.h
new file mode 100644
index 00000000..bc297870
--- /dev/null
+++ b/decoder/prob.h
@@ -0,0 +1,8 @@
+#ifndef _PROB_H_
+#define _PROB_H_
+
+#include "logval.h"
+
+typedef LogVal<double> prob_t;
+
+#endif
diff --git a/decoder/rule_lexer.h b/decoder/rule_lexer.h
new file mode 100644
index 00000000..e5db4018
--- /dev/null
+++ b/decoder/rule_lexer.h
@@ -0,0 +1,13 @@
+#ifndef _RULE_LEXER_H_
+#define _RULE_LEXER_H_
+
+#include <iostream>
+
+#include "trule.h"
+
+struct RuleLexer {
+  typedef void (*RuleCallback)(const TRulePtr& new_rule, void* extra);
+  static void ReadRules(std::istream* in, RuleCallback func, void* extra);
+};
+
+#endif
diff --git a/decoder/rule_lexer.l b/decoder/rule_lexer.l
new file mode 100644
index 00000000..ff8f10b0
--- /dev/null
+++ b/decoder/rule_lexer.l
@@ -0,0 +1,269 @@
+%{
+#include "rule_lexer.h"
+
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <cstring>
+#include <cassert>
+#include "tdict.h"
+#include "fdict.h"
+#include "trule.h"
+
+int lex_line = 0;
+std::istream* scfglex_stream = NULL;
+RuleLexer::RuleCallback rule_callback = NULL;
+void* rule_callback_extra = NULL;
+std::vector<int> scfglex_phrase_fnames;
+
+#undef YY_INPUT
+#define YY_INPUT(buf, result, max_size) (result = scfglex_stream->read(buf, max_size).gcount())
+
+#define YY_SKIP_YYWRAP 1
+int num_rules = 0;
+int yywrap() { return 1; }
+bool fl = true;
+#define MAX_TOKEN_SIZE 255
+std::string scfglex_tmp_token(MAX_TOKEN_SIZE, '\0');
+
+#define MAX_RULE_SIZE 48
+WordID scfglex_src_rhs[MAX_RULE_SIZE];
+WordID scfglex_trg_rhs[MAX_RULE_SIZE];
+int scfglex_src_rhs_size;
+int scfglex_trg_rhs_size;
+WordID scfglex_lhs;
+int scfglex_src_arity;
+int scfglex_trg_arity;
+
+#define MAX_FEATS 20
+int scfglex_feat_ids[MAX_FEATS];
+double scfglex_feat_vals[MAX_FEATS];
+int scfglex_num_feats;
+
+#define MAX_ARITY 20
+int scfglex_nt_sanity[MAX_ARITY];
+int scfglex_src_nts[MAX_ARITY];
+float scfglex_nt_size_means[MAX_ARITY];
+float scfglex_nt_size_vars[MAX_ARITY];
+
+
+void sanity_check_trg_symbol(WordID nt, int index) {
+  if (scfglex_src_nts[index-1] != nt) {
+    std::cerr << "Target symbol with index " << index << " is of type " << TD::Convert(nt*-1)
+              << " but corresponding source is of type "
+              << TD::Convert(scfglex_src_nts[index-1] * -1) << std::endl;
+    abort();
+  }
+}
+
+void sanity_check_trg_index(int index) {
+  if (index > scfglex_src_arity) {
+    std::cerr << "Target index " << index << " exceeds source arity " << scfglex_src_arity << std::endl;
+    abort();
+  }
+  int& flag = scfglex_nt_sanity[index - 1];
+  if (flag) {
+    std::cerr << "Target index " << index << " used multiple times!" << std::endl;
+    abort();
+  }
+  flag = 1;
+}
+
+void scfglex_reset() {
+  scfglex_src_arity = 0;
+  scfglex_trg_arity = 0;
+  scfglex_num_feats = 0;
+  scfglex_src_rhs_size = 0;
+  scfglex_trg_rhs_size = 0;
+}
+
+%}
+
+REAL [\-+]?[0-9]+(\.[0-9]*([eE][-+]*[0-9]+)?)?|inf|[\-+]inf
+NT [\-#$A-Z_:=.",\\][\-#$".A-Z+/=_0-9!:@\\]*
+
+%x LHS_END SRC TRG FEATS FEATVAL ALIGNS
+%%
+
+<INITIAL>[ \t]	;
+
+<INITIAL>\[{NT}\]   {
+		scfglex_tmp_token.assign(yytext + 1, yyleng - 2);
+		scfglex_lhs = -TD::Convert(scfglex_tmp_token);
+		// std::cerr << scfglex_tmp_token << "\n";
+  		BEGIN(LHS_END);
+		}
+
+<SRC>\[{NT}\]   {
+		scfglex_tmp_token.assign(yytext + 1, yyleng - 2);
+		scfglex_src_nts[scfglex_src_arity] = scfglex_src_rhs[scfglex_src_rhs_size] = -TD::Convert(scfglex_tmp_token);
+		++scfglex_src_arity;
+		++scfglex_src_rhs_size;
+		}
+
+<SRC>\[{NT},[1-9][0-9]?\]   {
+		int index = yytext[yyleng - 2] - '0';
+		if (yytext[yyleng - 3] == ',') {
+		  scfglex_tmp_token.assign(yytext + 1, yyleng - 4);
+		} else {
+		  scfglex_tmp_token.assign(yytext + 1, yyleng - 5);
+		  index += 10 * (yytext[yyleng - 3] - '0');
+		}
+		if ((scfglex_src_arity+1) != index) {
+			std::cerr << "Src indices must go in order: expected " << scfglex_src_arity << " but got " << index << std::endl;
+			abort();
+		}
+		scfglex_src_nts[scfglex_src_arity] = scfglex_src_rhs[scfglex_src_rhs_size] = -TD::Convert(scfglex_tmp_token);
+		++scfglex_src_rhs_size;
+		++scfglex_src_arity;
+		}
+
+<TRG>\[{NT},[1-9][0-9]?\]   {
+		int index = yytext[yyleng - 2] - '0';
+		if (yytext[yyleng - 3] == ',') {
+		  scfglex_tmp_token.assign(yytext + 1, yyleng - 4);
+		} else {
+		  scfglex_tmp_token.assign(yytext + 1, yyleng - 5);
+		  index += 10 * (yytext[yyleng - 3] - '0');
+		}
+		++scfglex_trg_arity;
+		// std::cerr << "TRG INDEX: " << index << std::endl;
+		sanity_check_trg_symbol(-TD::Convert(scfglex_tmp_token), index);
+		sanity_check_trg_index(index);
+		scfglex_trg_rhs[scfglex_trg_rhs_size] = 1 - index;
+		++scfglex_trg_rhs_size;
+}
+
+<TRG>\[[1-9][0-9]?\]   {
+		int index = yytext[yyleng - 2] - '0';
+		if (yyleng == 4) {
+		  index += 10 * (yytext[yyleng - 3] - '0');
+		}
+		++scfglex_trg_arity;
+		sanity_check_trg_index(index);
+		scfglex_trg_rhs[scfglex_trg_rhs_size] = 1 - index;
+		++scfglex_trg_rhs_size;
+}
+
+<LHS_END>[ \t] { ; }
+<LHS_END>\|\|\|	{
+		scfglex_reset();
+		BEGIN(SRC);
+		}
+<INITIAL,LHS_END>.	{
+		std::cerr << "Line " << lex_line << ": unexpected input in LHS: " << yytext << std::endl;
+		abort();
+		}
+
+<SRC>\|\|\|	{
+		memset(scfglex_nt_sanity, 0, scfglex_src_arity * sizeof(int));
+		BEGIN(TRG);
+		}
+<SRC>[^ \t]+	{ 
+		scfglex_tmp_token.assign(yytext, yyleng);
+		scfglex_src_rhs[scfglex_src_rhs_size] = TD::Convert(scfglex_tmp_token);
+		++scfglex_src_rhs_size;
+		}
+<SRC>[ \t]+	{ ; }
+
+<TRG>\|\|\|	{
+		BEGIN(FEATS);
+		}
+<TRG>[^ \t]+	{
+		scfglex_tmp_token.assign(yytext, yyleng);
+		scfglex_trg_rhs[scfglex_trg_rhs_size] = TD::Convert(scfglex_tmp_token);
+		++scfglex_trg_rhs_size;
+		}
+<TRG>[ \t]+	{ ; }
+
+<TRG,FEATS,ALIGNS>\n	{
+                if (scfglex_src_arity != scfglex_trg_arity) {
+                  std::cerr << "Line " << lex_line << ": LHS and RHS arity mismatch!\n";
+                  abort();
+                }
+		TRulePtr rp(new TRule(scfglex_lhs, scfglex_src_rhs, scfglex_src_rhs_size, scfglex_trg_rhs, scfglex_trg_rhs_size, scfglex_feat_ids, scfglex_feat_vals, scfglex_num_feats, scfglex_src_arity));
+		rule_callback(rp, rule_callback_extra);
+		// std::cerr << rp->AsString() << std::endl;
+		num_rules++;
+                lex_line++;
+                if (num_rules %   50000 == 0) { std::cerr << '.' << std::flush; fl = true; }
+                if (num_rules % 2000000 == 0) { std::cerr << " [" << num_rules << "]\n"; fl = false; }
+		BEGIN(INITIAL);
+		}
+
+<FEATS>[ \t;]	{ ; }
+<FEATS>[^ \t=;]+=	{
+		scfglex_tmp_token.assign(yytext, yyleng - 1);
+		const int fid = FD::Convert(scfglex_tmp_token);
+		if (fid < 1) {
+			std::cerr << "\nUNWEIGHED FEATURE " << scfglex_tmp_token << std::endl;
+			abort();
+		}
+		scfglex_feat_ids[scfglex_num_feats] = fid;
+		BEGIN(FEATVAL);
+		}
+<FEATS>\|\|\|	{
+		BEGIN(ALIGNS);
+		}
+<FEATVAL>{REAL}	{
+		scfglex_feat_vals[scfglex_num_feats] = strtod(yytext, NULL);
+		++scfglex_num_feats;
+		BEGIN(FEATS);
+		}
+<FEATVAL>.	{
+		std::cerr << "Line " << lex_line << ": unexpected input in feature value: " << yytext << std::endl;
+		abort();
+		}
+<FEATS>{REAL} 	{ 
+		scfglex_feat_ids[scfglex_num_feats] = scfglex_phrase_fnames[scfglex_num_feats];
+		scfglex_feat_vals[scfglex_num_feats] = strtod(yytext, NULL);
+		++scfglex_num_feats;
+		}
+<FEATS>.	{
+		std::cerr << "Line " << lex_line << " unexpected input in features: " << yytext << std::endl;
+		abort();
+		}
+<ALIGNS>[0-9]+-[0-9]+	{
+                int i = 0;
+		int a = 0;
+		int b = 0;
+		while (i < yyleng) {
+		  char c = yytext[i];
+		  if (c == '-') break;
+		  a *= 10;
+		  a += c - '0';
+		  ++i;
+		}
+		++i;
+		while (i < yyleng) {
+		  b *= 10;
+		  b += yytext[i] - '0';
+		  ++i;
+		}
+		// TODO store alignment points somewhere
+		}
+<ALIGNS>[ \t]	;
+<ALIGNS>.	{
+		std::cerr << "Line " << lex_line << ": unexpected input in alignment: " << yytext << std::endl;
+		abort();
+		}
+%%
+
+#include "filelib.h"
+
+void RuleLexer::ReadRules(std::istream* in, RuleLexer::RuleCallback func, void* extra) {
+  if (scfglex_phrase_fnames.empty()) {
+    scfglex_phrase_fnames.resize(100);
+    for (int i = 0; i < scfglex_phrase_fnames.size(); ++i) {
+      std::ostringstream os;
+      os << "PhraseModel_" << i;
+      scfglex_phrase_fnames[i] = FD::Convert(os.str());
+    }
+  }
+  lex_line = 1;
+  scfglex_stream = in;
+  rule_callback_extra = extra,
+  rule_callback = func;
+  yylex();
+}
+
diff --git a/decoder/sampler.h b/decoder/sampler.h
new file mode 100644
index 00000000..e5840f41
--- /dev/null
+++ b/decoder/sampler.h
@@ -0,0 +1,136 @@
+#ifndef SAMPLER_H_
+#define SAMPLER_H_
+
+#include <algorithm>
+#include <functional>
+#include <numeric>
+#include <iostream>
+#include <fstream>
+#include <vector>
+
+#include <boost/random/mersenne_twister.hpp>
+#include <boost/random/uniform_real.hpp>
+#include <boost/random/variate_generator.hpp>
+#include <boost/random/normal_distribution.hpp>
+#include <boost/random/poisson_distribution.hpp>
+
+#include "prob.h"
+
+struct SampleSet;
+
+template <typename RNG>
+struct RandomNumberGenerator {
+  static uint32_t GetTrulyRandomSeed() {
+    uint32_t seed;
+    std::ifstream r("/dev/urandom");
+    if (r) {
+      r.read((char*)&seed,sizeof(uint32_t));
+    }
+    if (r.fail() || !r) {
+      std::cerr << "Warning: could not read from /dev/urandom. Seeding from clock" << std::endl;
+      seed = time(NULL);
+    }
+    std::cerr << "Seeding random number sequence to " << seed << std::endl;
+    return seed;
+  }
+
+  RandomNumberGenerator() : m_dist(0,1), m_generator(), m_random(m_generator,m_dist) {
+    uint32_t seed = GetTrulyRandomSeed();
+    m_generator.seed(seed);
+  }
+  explicit RandomNumberGenerator(uint32_t seed) : m_dist(0,1), m_generator(), m_random(m_generator,m_dist) {
+    if (!seed) seed = GetTrulyRandomSeed();
+    m_generator.seed(seed);
+  }
+
+  size_t SelectSample(const prob_t& a, const prob_t& b, double T = 1.0) {
+    if (T == 1.0) {
+      if (this->next() > (a / (a + b))) return 1; else return 0;
+    } else {
+      assert(!"not implemented");
+    }
+  }
+
+  // T is the annealing temperature, if desired
+  size_t SelectSample(const SampleSet& ss, double T = 1.0);
+
+  // draw a value from U(0,1)
+  double next() {return m_random();}
+
+  // draw a value from N(mean,var)
+  double NextNormal(double mean, double var) {
+    return boost::normal_distribution<double>(mean, var)(m_random);
+  }
+
+  // draw a value from a Poisson distribution
+  // lambda must be greater than 0
+  int NextPoisson(int lambda) {
+    return boost::poisson_distribution<int>(lambda)(m_random);
+  }
+
+  bool AcceptMetropolisHastings(const prob_t& p_cur,
+                                const prob_t& p_prev,
+                                const prob_t& q_cur,
+                                const prob_t& q_prev) {
+    const prob_t a = (p_cur / p_prev) * (q_prev / q_cur);
+    if (log(a) >= 0.0) return true;
+    return (prob_t(this->next()) < a);
+  }
+
+ private:
+  boost::uniform_real<> m_dist;
+  RNG m_generator;
+  boost::variate_generator<RNG&, boost::uniform_real<> > m_random;
+};
+
+typedef RandomNumberGenerator<boost::mt19937> MT19937;
+
+class SampleSet {
+ public:
+  const prob_t& operator[](int i) const { return m_scores[i]; }
+  bool empty() const { return m_scores.empty(); }
+  void add(const prob_t& s) { m_scores.push_back(s); }
+  void clear() { m_scores.clear(); }
+  size_t size() const { return m_scores.size(); }
+  std::vector<prob_t> m_scores;
+};
+
+template <typename RNG>
+size_t RandomNumberGenerator<RNG>::SelectSample(const SampleSet& ss, double T) {
+  assert(T > 0.0);
+  assert(ss.m_scores.size() > 0);
+  if (ss.m_scores.size() == 1) return 0;
+  const prob_t annealing_factor(1.0 / T);
+  const bool anneal = (annealing_factor != prob_t::One());
+  prob_t sum = prob_t::Zero();
+  if (anneal) {
+    for (int i = 0; i < ss.m_scores.size(); ++i)
+      sum += ss.m_scores[i].pow(annealing_factor);  // p^(1/T)
+  } else {
+    sum = std::accumulate(ss.m_scores.begin(), ss.m_scores.end(), prob_t::Zero());
+  }
+  //for (size_t i = 0; i < ss.m_scores.size(); ++i) std::cerr << ss.m_scores[i] << ",";
+  //std::cerr << std::endl;
+ 
+  prob_t random(this->next());    // random number between 0 and 1
+  random *= sum;                  // scale with normalization factor
+  //std::cerr << "Random number " << random << std::endl;
+
+  //now figure out which sample
+  size_t position = 1;
+  sum = ss.m_scores[0];
+  if (anneal) {
+    sum.poweq(annealing_factor);
+    for (; position < ss.m_scores.size() && sum < random; ++position) 
+      sum += ss.m_scores[position].pow(annealing_factor);
+  } else {
+    for (; position < ss.m_scores.size() && sum < random; ++position) 
+      sum += ss.m_scores[position];
+  }
+  //std::cout << "random: " << random <<  " sample: " << position << std::endl;
+  //std::cerr << "Sample: " << position-1 << std::endl;
+  //exit(1);
+  return position-1;
+}
+
+#endif
diff --git a/decoder/scfg_translator.cc b/decoder/scfg_translator.cc
new file mode 100644
index 00000000..c215eea6
--- /dev/null
+++ b/decoder/scfg_translator.cc
@@ -0,0 +1,132 @@
+#include "translator.h"
+
+#include <vector>
+
+#include "hg.h"
+#include "grammar.h"
+#include "bottom_up_parser.h"
+#include "sentence_metadata.h"
+
+using namespace std;
+static bool usingSentenceGrammar = false;
+static bool printGrammarsUsed = false;
+
+struct SCFGTranslatorImpl {
+  SCFGTranslatorImpl(const boost::program_options::variables_map& conf) :
+      max_span_limit(conf["scfg_max_span_limit"].as<int>()),
+      add_pass_through_rules(conf.count("add_pass_through_rules")),
+      goal(conf["goal"].as<string>()),
+      default_nt(conf["scfg_default_nt"].as<string>()) {
+    if(conf.count("grammar"))
+      {
+	vector<string> gfiles = conf["grammar"].as<vector<string> >();
+	for (int i = 0; i < gfiles.size(); ++i) {
+	  cerr << "Reading SCFG grammar from " << gfiles[i] << endl;
+	  TextGrammar* g = new TextGrammar(gfiles[i]);
+	  g->SetMaxSpan(max_span_limit);
+	  g->SetGrammarName(gfiles[i]);
+	  grammars.push_back(GrammarPtr(g));
+	  
+	}
+      }
+    if (!conf.count("scfg_no_hiero_glue_grammar"))
+      { 
+	GlueGrammar* g = new GlueGrammar(goal, default_nt);
+	g->SetGrammarName("GlueGrammar");
+	grammars.push_back(GrammarPtr(g));
+	cerr << "Adding glue grammar" << endl;
+      }
+    if (conf.count("scfg_extra_glue_grammar"))
+      {
+	GlueGrammar* g = new GlueGrammar(conf["scfg_extra_glue_grammar"].as<string>());
+	g->SetGrammarName("ExtraGlueGrammar");		
+	grammars.push_back(GrammarPtr(g));
+	cerr << "Adding extra glue grammar" << endl;
+      }
+  }
+
+  const int max_span_limit;
+  const bool add_pass_through_rules;
+  const string goal;
+  const string default_nt;
+  vector<GrammarPtr> grammars;
+
+  bool Translate(const string& input,
+                 SentenceMetadata* smeta,
+                 const vector<double>& weights,
+                 Hypergraph* forest) {
+    vector<GrammarPtr> glist = grammars;
+    Lattice& lattice = smeta->src_lattice_;
+    LatticeTools::ConvertTextOrPLF(input, &lattice);
+    smeta->SetSourceLength(lattice.size());
+    if (add_pass_through_rules){
+      PassThroughGrammar* g = new PassThroughGrammar(lattice, default_nt);
+      g->SetGrammarName("PassThrough");
+      glist.push_back(GrammarPtr(g));
+      cerr << "Adding pass through grammar" << endl;
+    }
+
+
+
+    if(printGrammarsUsed){    //Iterate trough grammars we have for this sentence and list them
+      for (int gi = 0; gi < glist.size(); ++gi) 
+	{
+	  cerr << "Using grammar::" << 	 glist[gi]->GetGrammarName() << endl;
+	}
+    }
+
+    ExhaustiveBottomUpParser parser(goal, glist);
+    if (!parser.Parse(lattice, forest))
+      return false;
+    forest->Reweight(weights);
+    return true;
+  }
+};
+
+/*
+Called once from cdec.cc to setup the initial SCFG translation structure backend
+*/
+SCFGTranslator::SCFGTranslator(const boost::program_options::variables_map& conf) :
+  pimpl_(new SCFGTranslatorImpl(conf)) {}
+
+/*
+Called for each sentence to perform translation using the SCFG backend
+*/
+bool SCFGTranslator::TranslateImpl(const string& input,
+                               SentenceMetadata* smeta,
+                               const vector<double>& weights,
+                               Hypergraph* minus_lm_forest) {
+  
+  return pimpl_->Translate(input, smeta, weights, minus_lm_forest);
+}
+
+/*
+Check for grammar pointer in the sentence markup, for use with sentence specific grammars
+ */
+void SCFGTranslator::ProcessMarkupHintsImpl(const map<string, string>& kv) {
+  map<string,string>::const_iterator it = kv.find("grammar");
+ 
+  
+  if (it == kv.end()) {
+    usingSentenceGrammar= false; 
+    return;
+  }
+  //Create sentence specific grammar from specified file name and load grammar into list of grammars
+  cerr << "Loading sentence grammar from:" << it->second <<  endl;
+  usingSentenceGrammar = true;
+  TextGrammar* sentGrammar = new TextGrammar(it->second);
+  sentGrammar->SetMaxSpan(pimpl_->max_span_limit);
+  sentGrammar->SetGrammarName(it->second);
+  pimpl_->grammars.push_back(GrammarPtr(sentGrammar));
+
+}
+
+void SCFGTranslator::SentenceCompleteImpl() {
+
+  if(usingSentenceGrammar)      // Drop the last sentence grammar from the list of grammars
+    {
+      cerr << "Clearing grammar" << endl;
+      pimpl_->grammars.pop_back();
+    }
+}
+
diff --git a/decoder/sentence_metadata.h b/decoder/sentence_metadata.h
new file mode 100644
index 00000000..ef9eb388
--- /dev/null
+++ b/decoder/sentence_metadata.h
@@ -0,0 +1,47 @@
+#ifndef _SENTENCE_METADATA_H_
+#define _SENTENCE_METADATA_H_
+
+#include <cassert>
+#include "lattice.h"
+
+struct SentenceMetadata {
+  SentenceMetadata(int id, const Lattice& ref) :
+    sent_id_(id),
+    src_len_(-1),
+    has_reference_(ref.size() > 0),
+    trg_len_(ref.size()),
+    ref_(has_reference_ ? &ref : NULL) {}
+
+  // this should be called by the Translator object after
+  // it has parsed the source
+  void SetSourceLength(int sl) { src_len_ = sl; }
+
+  // this should be called if a separate model needs to
+  // specify how long the target sentence should be
+  void SetTargetLength(int tl) {
+    assert(!has_reference_);
+    trg_len_ = tl;
+  }
+  bool HasReference() const { return has_reference_; }
+  const Lattice& GetReference() const { return *ref_; }
+  int GetSourceLength() const { return src_len_; }
+  int GetTargetLength() const { return trg_len_; }
+  int GetSentenceID() const { return sent_id_; }
+  // this will be empty if the translator accepts non FS input!
+  const Lattice& GetSourceLattice() const { return src_lattice_; }
+
+ private:
+  const int sent_id_;
+  // the following should be set, if possible, by the Translator
+  int src_len_;
+ public:
+  Lattice src_lattice_;  // this will only be set if inputs are finite state!
+ private:
+  // you need to be very careful when depending on these values
+  // they will only be set during training / alignment contexts
+  const bool has_reference_;
+  int trg_len_;
+  const Lattice* const ref_;
+};
+
+#endif
diff --git a/decoder/small_vector.h b/decoder/small_vector.h
new file mode 100644
index 00000000..800c1df1
--- /dev/null
+++ b/decoder/small_vector.h
@@ -0,0 +1,187 @@
+#ifndef _SMALL_VECTOR_H_
+
+#include <streambuf>  // std::max - where to get this?
+#include <cstring>
+#include <cassert>
+
+#define __SV_MAX_STATIC 2
+
+class SmallVector {
+
+ public:
+  SmallVector() : size_(0) {}
+
+  explicit SmallVector(size_t s, int v = 0) : size_(s) {
+    assert(s < 0x80);
+    if (s <= __SV_MAX_STATIC) {
+      for (int i = 0; i < s; ++i) data_.vals[i] = v;
+    } else {
+      capacity_ = s;
+      size_ = s;
+      data_.ptr = new int[s];
+      for (int i = 0; i < size_; ++i) data_.ptr[i] = v;
+    }
+  }
+
+  SmallVector(const SmallVector& o) : size_(o.size_) {
+    if (size_ <= __SV_MAX_STATIC) {
+      for (int i = 0; i < __SV_MAX_STATIC; ++i) data_.vals[i] = o.data_.vals[i];
+    } else {
+      capacity_ = size_ = o.size_;
+      data_.ptr = new int[capacity_];
+      std::memcpy(data_.ptr, o.data_.ptr, size_ * sizeof(int));
+    }
+  }
+
+  const SmallVector& operator=(const SmallVector& o) {
+    if (size_ <= __SV_MAX_STATIC) {
+      if (o.size_ <= __SV_MAX_STATIC) {
+        size_ = o.size_;
+        for (int i = 0; i < __SV_MAX_STATIC; ++i) data_.vals[i] = o.data_.vals[i];
+      } else {
+        capacity_ = size_ = o.size_;
+        data_.ptr = new int[capacity_];
+        std::memcpy(data_.ptr, o.data_.ptr, size_ * sizeof(int));
+      }
+    } else {
+      if (o.size_ <= __SV_MAX_STATIC) {
+        delete[] data_.ptr;
+        size_ = o.size_;
+        for (int i = 0; i < size_; ++i) data_.vals[i] = o.data_.vals[i];
+      } else {
+        if (capacity_ < o.size_) {
+          delete[] data_.ptr;
+          capacity_ = o.size_;
+          data_.ptr = new int[capacity_];
+        }
+        size_ = o.size_;
+        for (int i = 0; i < size_; ++i)
+          data_.ptr[i] = o.data_.ptr[i];
+      }
+    }
+    return *this;
+  }
+
+  ~SmallVector() {
+    if (size_ <= __SV_MAX_STATIC) return;
+    delete[] data_.ptr;
+  }
+
+  void clear() {
+    if (size_ > __SV_MAX_STATIC) {
+      delete[] data_.ptr;
+    }
+    size_ = 0;
+  }
+
+  bool empty() const { return size_ == 0; }
+  size_t size() const { return size_; }
+
+  inline void ensure_capacity(unsigned char min_size) {
+    assert(min_size > __SV_MAX_STATIC);
+    if (min_size < capacity_) return;
+    unsigned char new_cap = std::max(static_cast<unsigned char>(capacity_ << 1), min_size);
+    int* tmp = new int[new_cap];
+    std::memcpy(tmp, data_.ptr, capacity_ * sizeof(int));
+    delete[] data_.ptr;
+    data_.ptr = tmp;
+    capacity_ = new_cap;
+  }
+
+  inline void copy_vals_to_ptr() {
+    capacity_ = __SV_MAX_STATIC * 2;
+    int* tmp = new int[capacity_];
+    for (int i = 0; i < __SV_MAX_STATIC; ++i) tmp[i] = data_.vals[i];
+    data_.ptr = tmp;
+  }
+
+  inline void push_back(int v) {
+    if (size_ < __SV_MAX_STATIC) {
+      data_.vals[size_] = v;
+      ++size_;
+      return;
+    } else if (size_ == __SV_MAX_STATIC) {
+      copy_vals_to_ptr();
+    } else if (size_ == capacity_) {
+      ensure_capacity(size_ + 1);
+    }
+    data_.ptr[size_] = v;
+    ++size_;
+  }
+
+  int& back() { return this->operator[](size_ - 1); }
+  const int& back() const { return this->operator[](size_ - 1); }
+  int& front() { return this->operator[](0); }
+  const int& front() const { return this->operator[](0); }
+
+  void resize(size_t s, int v = 0) {
+    if (s <= __SV_MAX_STATIC) {
+      if (size_ > __SV_MAX_STATIC) {
+        int tmp[__SV_MAX_STATIC];
+        for (int i = 0; i < s; ++i) tmp[i] = data_.ptr[i];
+        delete[] data_.ptr;
+        for (int i = 0; i < s; ++i) data_.vals[i] = tmp[i];
+        size_ = s;
+        return;
+      }
+      if (s <= size_) {
+        size_ = s;
+        return;
+      } else {
+        for (int i = size_; i < s; ++i)
+          data_.vals[i] = v;
+        size_ = s;
+        return;
+      }
+    } else {
+      if (size_ <= __SV_MAX_STATIC)
+        copy_vals_to_ptr();
+      if (s > capacity_)
+        ensure_capacity(s);
+      if (s > size_) {
+        for (int i = size_; i < s; ++i)
+          data_.ptr[i] = v;
+      }
+      size_ = s;
+    }
+  }
+
+  int& operator[](size_t i) {
+    if (size_ <= __SV_MAX_STATIC) return data_.vals[i];
+    return data_.ptr[i];
+  }
+
+  const int& operator[](size_t i) const {
+    if (size_ <= __SV_MAX_STATIC) return data_.vals[i];
+    return data_.ptr[i];
+  }
+
+  bool operator==(const SmallVector& o) const {
+    if (size_ != o.size_) return false;
+    if (size_ <= __SV_MAX_STATIC) {
+      for (size_t i = 0; i < size_; ++i)
+        if (data_.vals[i] != o.data_.vals[i]) return false;
+      return true;
+    } else {
+      for (size_t i = 0; i < size_; ++i)
+        if (data_.ptr[i] != o.data_.ptr[i]) return false;
+      return true;
+    }
+  }
+
+ private:
+  unsigned char capacity_;  // only defined when size_ >= __SV_MAX_STATIC
+  unsigned char size_;
+  union StorageType {
+    int vals[__SV_MAX_STATIC];
+    int* ptr;
+  };
+  StorageType data_;
+
+};
+
+inline bool operator!=(const SmallVector& a, const SmallVector& b) {
+  return !(a==b);
+}
+
+#endif
diff --git a/decoder/small_vector_test.cc b/decoder/small_vector_test.cc
new file mode 100644
index 00000000..84237791
--- /dev/null
+++ b/decoder/small_vector_test.cc
@@ -0,0 +1,129 @@
+#include "small_vector.h"
+
+#include <gtest/gtest.h>
+#include <iostream>
+#include <cassert>
+#include <vector>
+
+using namespace std;
+
+class SVTest : public testing::Test {
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+};
+       
+TEST_F(SVTest, LargerThan2) {
+  SmallVector v;
+  SmallVector v2;
+  v.push_back(0);
+  v.push_back(1);
+  v.push_back(2);
+  assert(v.size() == 3);
+  assert(v[2] == 2);
+  assert(v[1] == 1);
+  assert(v[0] == 0);
+  v2 = v;
+  SmallVector copy(v);
+  assert(copy.size() == 3);
+  assert(copy[0] == 0);
+  assert(copy[1] == 1);
+  assert(copy[2] == 2);
+  assert(copy == v2);
+  copy[1] = 99;
+  assert(copy != v2);
+  assert(v2.size() == 3);
+  assert(v2[2] == 2);
+  assert(v2[1] == 1);
+  assert(v2[0] == 0);
+  v2[0] = -2;
+  v2[1] = -1;
+  v2[2] = 0;
+  assert(v2[2] == 0);
+  assert(v2[1] == -1);
+  assert(v2[0] == -2);
+  SmallVector v3(1,1);
+  assert(v3[0] == 1);
+  v2 = v3;
+  assert(v2.size() == 1);
+  assert(v2[0] == 1);
+  SmallVector v4(10, 1);
+  assert(v4.size() == 10);
+  assert(v4[5] == 1);
+  assert(v4[9] == 1);
+  v4 = v;
+  assert(v4.size() == 3);
+  assert(v4[2] == 2);
+  assert(v4[1] == 1);
+  assert(v4[0] == 0);
+  SmallVector v5(10, 2);
+  assert(v5.size() == 10);
+  assert(v5[7] == 2);
+  assert(v5[0] == 2);
+  assert(v.size() == 3);
+  v = v5;
+  assert(v.size() == 10);
+  assert(v[2] == 2);
+  assert(v[9] == 2);
+  SmallVector cc;
+  for (int i = 0; i < 33; ++i)
+    cc.push_back(i);
+  for (int i = 0; i < 33; ++i)
+    assert(cc[i] == i);
+  cc.resize(20);
+  assert(cc.size() == 20);
+  for (int i = 0; i < 20; ++i)
+    assert(cc[i] == i);
+  cc[0]=-1;
+  cc.resize(1, 999);
+  assert(cc.size() == 1);
+  assert(cc[0] == -1);
+  cc.resize(99, 99);
+  for (int i = 1; i < 99; ++i) {
+    cerr << i << " " << cc[i] << endl;
+    assert(cc[i] == 99);
+  }
+  cc.clear();
+  assert(cc.size() == 0);
+}
+
+TEST_F(SVTest, Small) {
+  SmallVector v;
+  SmallVector v1(1,0);
+  SmallVector v2(2,10);
+  SmallVector v1a(2,0);
+  EXPECT_TRUE(v1 != v1a);
+  EXPECT_TRUE(v1 == v1);
+  EXPECT_EQ(v1[0], 0);
+  EXPECT_EQ(v2[1], 10);
+  EXPECT_EQ(v2[0], 10);
+  ++v2[1];
+  --v2[0];
+  EXPECT_EQ(v2[0], 9);
+  EXPECT_EQ(v2[1], 11);
+  SmallVector v3(v2);
+  assert(v3[0] == 9);
+  assert(v3[1] == 11);
+  assert(!v3.empty());
+  assert(v3.size() == 2);
+  v3.clear();
+  assert(v3.empty());
+  assert(v3.size() == 0);
+  assert(v3 != v2);
+  assert(v2 != v3);
+  v3 = v2;
+  assert(v3 == v2);
+  assert(v2 == v3);
+  assert(v3[0] == 9);
+  assert(v3[1] == 11);
+  assert(!v3.empty());
+  assert(v3.size() == 2);
+  cerr << sizeof(SmallVector) << endl;
+  cerr << sizeof(vector<int>) << endl;
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
diff --git a/decoder/sparse_vector.cc b/decoder/sparse_vector.cc
new file mode 100644
index 00000000..4035b9ef
--- /dev/null
+++ b/decoder/sparse_vector.cc
@@ -0,0 +1,98 @@
+#include "sparse_vector.h"
+
+#include <iostream>
+#include <cstring>
+
+#include "hg_io.h"
+
+using namespace std;
+
+namespace B64 {
+
+void Encode(double objective, const SparseVector<double>& v, ostream* out) {
+  const int num_feats = v.num_active();
+  size_t tot_size = 0;
+  const size_t off_objective = tot_size;
+  tot_size += sizeof(double);                   // objective
+  const size_t off_num_feats = tot_size;
+  tot_size += sizeof(int);                      // num_feats
+  const size_t off_data = tot_size;
+  tot_size += sizeof(unsigned char) * num_feats; // lengths of feature names;
+  typedef SparseVector<double>::const_iterator const_iterator;
+  for (const_iterator it = v.begin(); it != v.end(); ++it)
+    tot_size += FD::Convert(it->first).size();   // feature names;
+  tot_size += sizeof(double) * num_feats;        // gradient
+  const size_t off_magic = tot_size;
+  tot_size += 4;                                 // magic
+
+  // size_t b64_size = tot_size * 4 / 3;
+  // cerr << "Sparse vector binary size: " << tot_size << "  (b64 size=" << b64_size << ")\n";
+  char* data = new char[tot_size];
+  *reinterpret_cast<double*>(&data[off_objective]) = objective;
+  *reinterpret_cast<int*>(&data[off_num_feats]) = num_feats;
+  char* cur = &data[off_data];
+  assert(cur - data == off_data);
+  for (const_iterator it = v.begin(); it != v.end(); ++it) {
+    const string& fname = FD::Convert(it->first);
+    *cur++ = static_cast<char>(fname.size());   // name len
+    memcpy(cur, &fname[0], fname.size());
+    cur += fname.size();
+    *reinterpret_cast<double*>(cur) = it->second;
+    cur += sizeof(double);
+  }
+  assert(cur - data == off_magic);
+  *reinterpret_cast<unsigned int*>(cur) = 0xBAABABBAu;
+  cur += sizeof(unsigned int);
+  assert(cur - data == tot_size);
+  b64encode(data, tot_size, out);
+  delete[] data;
+}
+
+bool Decode(double* objective, SparseVector<double>* v, const char* in, size_t size) {
+  v->clear();
+  if (size % 4 != 0) {
+    cerr << "B64 error - line % 4 != 0\n";
+    return false;
+  }
+  const size_t decoded_size = size * 3 / 4 - sizeof(unsigned int);
+  const size_t buf_size = decoded_size + sizeof(unsigned int);
+  if (decoded_size < 6) { cerr << "SparseVector decoding error: too short!\n"; return false; }
+  char* data = new char[buf_size];
+  if (!b64decode(reinterpret_cast<const unsigned char*>(in), size, data, buf_size)) {
+    delete[] data;
+    return false;
+  }
+  size_t cur = 0;
+  *objective = *reinterpret_cast<double*>(data);
+  cur += sizeof(double);
+  const int num_feats = *reinterpret_cast<int*>(&data[cur]);
+  cur += sizeof(int);
+  int fc = 0;
+  while(fc < num_feats && cur < decoded_size) {
+    ++fc;
+    const int fname_len = data[cur++];
+    assert(fname_len > 0);
+    assert(fname_len < 256);
+    string fname(fname_len, '\0');
+    memcpy(&fname[0], &data[cur], fname_len);
+    cur += fname_len;
+    const double val = *reinterpret_cast<double*>(&data[cur]);
+    cur += sizeof(double);
+    int fid = FD::Convert(fname);
+    v->set_value(fid, val);
+  }
+  if(num_feats != fc) {
+    cerr << "Expected " << num_feats << " but only decoded " << fc << "!\n";
+    delete[] data;
+    return false;
+  }
+  if (*reinterpret_cast<unsigned int*>(&data[cur]) != 0xBAABABBAu) {
+    cerr << "SparseVector decodeding error : magic does not match!\n";
+    delete[] data;
+    return false;
+  }
+  delete[] data;
+  return true;
+}
+
+}
diff --git a/decoder/sparse_vector.h b/decoder/sparse_vector.h
new file mode 100644
index 00000000..66c9b10d
--- /dev/null
+++ b/decoder/sparse_vector.h
@@ -0,0 +1,274 @@
+#ifndef _SPARSE_VECTOR_H_
+#define _SPARSE_VECTOR_H_
+
+// this is a modified version of code originally written
+// by Phil Blunsom
+
+#include <iostream>
+#include <map>
+#include <tr1/unordered_map>
+#include <vector>
+#include <valarray>
+
+#include "fdict.h"
+
+template <typename T>
+class SparseVector {
+public:
+  typedef std::map<int, T> MapType;
+  typedef typename std::map<int, T>::const_iterator const_iterator;
+  SparseVector() {}
+
+  const T operator[](int index) const {
+    typename MapType::const_iterator found = values_.find(index);
+    if (found == values_.end())
+      return T(0);
+    else
+      return found->second;
+  }
+
+  void set_value(int index, const T &value) {
+    values_[index] = value;
+  }
+
+    T add_value(int index, const T &value) {
+        return values_[index] += value;
+    }
+
+    T value(int index) const {
+        typename MapType::const_iterator found = values_.find(index);
+        if (found != values_.end())
+            return found->second;
+        else
+            return T(0);
+    }
+
+    void store(std::valarray<T>* target) const {
+      (*target) *= 0;
+      for (typename MapType::const_iterator 
+              it = values_.begin(); it != values_.end(); ++it) {
+        if (it->first >= target->size()) break;
+        (*target)[it->first] = it->second;
+      }
+    }
+
+    int max_index() const {
+        if (values_.empty()) return 0;
+        typename MapType::const_iterator found =values_.end();
+        --found;
+        return found->first;
+    }
+
+    // dot product with a unit vector of the same length
+    // as the sparse vector
+    T dot() const {
+        T sum = 0;
+        for (typename MapType::const_iterator 
+                it = values_.begin(); it != values_.end(); ++it)
+            sum += it->second;
+        return sum;
+    }
+
+    template<typename S>
+    S dot(const SparseVector<S> &vec) const {
+        S sum = 0;
+        for (typename MapType::const_iterator 
+                it = values_.begin(); it != values_.end(); ++it)
+        {
+            typename MapType::const_iterator 
+                found = vec.values_.find(it->first);
+            if (found != vec.values_.end())
+                sum += it->second * found->second;
+        }
+        return sum;
+    }
+    
+    template<typename S>
+    S dot(const std::vector<S> &vec) const {
+        S sum = 0;
+        for (typename MapType::const_iterator 
+                it = values_.begin(); it != values_.end(); ++it)
+        {
+            if (it->first < static_cast<int>(vec.size()))
+                sum += it->second * vec[it->first];
+        }
+        return sum;
+    }
+
+    template<typename S>
+    S dot(const S *vec) const {
+        // this is not range checked!
+        S sum = 0;
+        for (typename MapType::const_iterator 
+                it = values_.begin(); it != values_.end(); ++it)
+            sum += it->second * vec[it->first];
+        std::cout << "dot(*vec) " << sum << std::endl;
+        return sum;
+    }
+
+    T l1norm() const {
+      T sum = 0;
+      for (typename MapType::const_iterator 
+              it = values_.begin(); it != values_.end(); ++it)
+        sum += fabs(it->second);
+      return sum;
+    }
+    
+    T l2norm() const {
+      T sum = 0;
+      for (typename MapType::const_iterator 
+              it = values_.begin(); it != values_.end(); ++it)
+        sum += it->second * it->second;
+      return sqrt(sum);
+    }
+    
+    SparseVector<T> &operator+=(const SparseVector<T> &other) {
+        for (typename MapType::const_iterator 
+                it = other.values_.begin(); it != other.values_.end(); ++it)
+        {
+            T v = (values_[it->first] += it->second);
+            if (v == T())
+                values_.erase(it->first);
+        }
+        return *this;
+    }
+
+    SparseVector<T> &operator-=(const SparseVector<T> &other) {
+        for (typename MapType::const_iterator 
+                it = other.values_.begin(); it != other.values_.end(); ++it)
+        {
+            T v = (values_[it->first] -= it->second);
+            if (v == T(0))
+                values_.erase(it->first);
+        }
+        return *this;
+    }
+
+    SparseVector<T> &operator-=(const double &x) {
+        for (typename MapType::iterator 
+                it = values_.begin(); it != values_.end(); ++it)
+            it->second -= x;
+        return *this;
+    }
+
+    SparseVector<T> &operator+=(const double &x) {
+        for (typename MapType::iterator 
+                it = values_.begin(); it != values_.end(); ++it)
+            it->second += x;
+        return *this;
+    }
+
+    SparseVector<T> &operator/=(const T &x) {
+        for (typename MapType::iterator 
+                it = values_.begin(); it != values_.end(); ++it)
+            it->second /= x;
+        return *this;
+    }
+
+    SparseVector<T> &operator*=(const T& x) {
+        for (typename MapType::iterator 
+                it = values_.begin(); it != values_.end(); ++it)
+            it->second *= x;
+        return *this;
+    }
+
+    SparseVector<T> operator+(const double &x) const {
+        SparseVector<T> result = *this;
+        return result += x;
+    }
+
+    SparseVector<T> operator-(const double &x) const {
+        SparseVector<T> result = *this;
+        return result -= x;
+    }
+
+    SparseVector<T> operator/(const double &x) const {
+        SparseVector<T> result = *this;
+        return result /= x;
+    }
+
+    std::ostream &operator<<(std::ostream &out) const {
+        bool first = true;
+        for (typename MapType::const_iterator 
+                it = values_.begin(); it != values_.end(); ++it) {
+          // by definition feature id 0 is a dummy value
+          if (it->first == 0) continue;
+          out << (first ? "" : ";")
+	      << FD::Convert(it->first) << '=' << it->second;
+          first = false;
+        }
+        return out;
+    }
+
+    bool operator<(const SparseVector<T> &other) const {
+        typename MapType::const_iterator it = values_.begin();
+        typename MapType::const_iterator other_it = other.values_.begin();
+
+        for (; it != values_.end() && other_it != other.values_.end(); ++it, ++other_it)
+        {
+            if (it->first < other_it->first) return true;
+            if (it->first > other_it->first) return false;
+            if (it->second < other_it->second) return true;
+            if (it->second > other_it->second) return false;
+        }
+        return values_.size() < other.values_.size();
+    }
+
+    int num_active() const { return values_.size(); }
+    bool empty() const { return values_.empty(); }
+
+    const_iterator begin() const { return values_.begin(); }
+    const_iterator end() const { return values_.end(); }
+
+    void clear() {
+        values_.clear();
+    }
+    void clear_value(int index) {
+      values_.erase(index);
+    }
+
+    void swap(SparseVector<T>& other) {
+      values_.swap(other.values_);
+    }
+
+private:
+  MapType values_;
+};
+
+template <typename T>
+SparseVector<T> operator+(const SparseVector<T>& a, const SparseVector<T>& b) {
+  SparseVector<T> result = a;
+  return result += b;
+}
+
+template <typename T>
+SparseVector<T> operator*(const SparseVector<T>& a, const double& b) {
+  SparseVector<T> result = a;
+  return result *= b;
+}
+
+template <typename T>
+SparseVector<T> operator*(const SparseVector<T>& a, const T& b) {
+  SparseVector<T> result = a;
+  return result *= b;
+}
+
+template <typename T>
+SparseVector<T> operator*(const double& a, const SparseVector<T>& b) {
+  SparseVector<T> result = b;
+  return result *= a;
+}
+
+template <typename T>
+std::ostream &operator<<(std::ostream &out, const SparseVector<T> &vec)
+{
+    return vec.operator<<(out);
+}
+
+namespace B64 {
+  void Encode(double objective, const SparseVector<double>& v, std::ostream* out);
+  // returns false if failed to decode
+  bool Decode(double* objective, SparseVector<double>* v, const char* data, size_t size);
+}
+
+#endif
diff --git a/decoder/stringlib.cc b/decoder/stringlib.cc
new file mode 100644
index 00000000..3e52ae87
--- /dev/null
+++ b/decoder/stringlib.cc
@@ -0,0 +1,98 @@
+#include "stringlib.h"
+
+#include <cstring>
+#include <cstdlib>
+#include <cassert>
+#include <iostream>
+#include <map>
+
+#include "lattice.h"
+
+using namespace std;
+
+void ParseTranslatorInput(const string& line, string* input, string* ref) {
+  size_t hint = 0;
+  if (line.find("{\"rules\":") == 0) {
+    hint = line.find("}}");
+    if (hint == string::npos) {
+      cerr << "Syntax error: " << line << endl;
+      abort();
+    }
+    hint += 2;
+  }
+  size_t pos = line.find("|||", hint);
+  if (pos == string::npos) { *input = line; return; }
+  ref->clear();
+  *input = line.substr(0, pos - 1);
+  string rline = line.substr(pos + 4);
+  if (rline.size() > 0) {
+    assert(ref);
+    *ref = rline;
+  }
+}
+
+void ParseTranslatorInputLattice(const string& line, string* input, Lattice* ref) {
+  string sref;
+  ParseTranslatorInput(line, input, &sref);
+  if (sref.size() > 0) {
+    assert(ref);
+    LatticeTools::ConvertTextOrPLF(sref, ref);
+  }
+}
+
+void ProcessAndStripSGML(string* pline, map<string, string>* out) {
+  map<string, string>& meta = *out;
+  string& line = *pline;
+  string lline = LowercaseString(line);
+  if (lline.find("<seg")!=0) return;
+  size_t close = lline.find(">");
+  if (close == string::npos) return; // error
+  size_t end = lline.find("</seg>");
+  string seg = Trim(lline.substr(4, close-4));
+  string text = line.substr(close+1, end - close - 1);
+  for (size_t i = 1; i < seg.size(); i++) {
+    if (seg[i] == '=' && seg[i-1] == ' ') {
+      string less = seg.substr(0, i-1) + seg.substr(i);
+      seg = less; i = 0; continue;
+    }
+    if (seg[i] == '=' && seg[i+1] == ' ') {
+      string less = seg.substr(0, i+1);
+      if (i+2 < seg.size()) less += seg.substr(i+2);
+      seg = less; i = 0; continue;
+    }
+  }
+  line = Trim(text);
+  if (seg == "") return;
+  for (size_t i = 1; i < seg.size(); i++) {
+    if (seg[i] == '=') {
+      string label = seg.substr(0, i);
+      string val = seg.substr(i+1);
+      if (val[0] == '"') {
+        val = val.substr(1);
+        size_t close = val.find('"');
+        if (close == string::npos) {
+          cerr << "SGML parse error: missing \"\n";
+          seg = "";
+          i = 0;
+        } else {
+          seg = val.substr(close+1);
+          val = val.substr(0, close);
+          i = 0;
+        }
+      } else {
+        size_t close = val.find(' ');
+        if (close == string::npos) {
+          seg = "";
+          i = 0;
+        } else {
+          seg = val.substr(close+1);
+          val = val.substr(0, close);
+        }
+      }
+      label = Trim(label);
+      seg = Trim(seg);
+      meta[label] = val;
+    }
+  }
+}
+
diff --git a/decoder/stringlib.h b/decoder/stringlib.h
new file mode 100644
index 00000000..76efee8f
--- /dev/null
+++ b/decoder/stringlib.h
@@ -0,0 +1,101 @@
+#ifndef _STRINGLIB_H_
+
+#include <map>
+#include <vector>
+#include <cctype>
+#include <string>
+
+// read line in the form of either:
+//   source
+//   source ||| target
+// source will be returned as a string, target must be a sentence or
+// a lattice (in PLF format) and will be returned as a Lattice object
+void ParseTranslatorInput(const std::string& line, std::string* input, std::string* ref);
+struct Lattice;
+void ParseTranslatorInputLattice(const std::string& line, std::string* input, Lattice* ref);
+
+inline const std::string Trim(const std::string& str, const std::string& dropChars = " \t") {
+  std::string res = str;
+  res.erase(str.find_last_not_of(dropChars)+1);
+  return res.erase(0, res.find_first_not_of(dropChars));
+}
+
+inline void Tokenize(const std::string& str, char delimiter, std::vector<std::string>* res) {
+  std::string s = str;
+  int last = 0;
+  res->clear();
+  for (int i=0; i < s.size(); ++i)
+    if (s[i] == delimiter) {
+      s[i]=0;
+      if (last != i) {
+        res->push_back(&s[last]);
+      }
+      last = i + 1;
+    }
+  if (last != s.size())
+    res->push_back(&s[last]);
+}
+
+inline std::string LowercaseString(const std::string& in) {
+  std::string res(in.size(),' ');
+  for (int i = 0; i < in.size(); ++i)
+    res[i] = tolower(in[i]);
+  return res;
+}
+
+inline int CountSubstrings(const std::string& str, const std::string& sub) {
+  size_t p = 0;
+  int res = 0;
+  while (p < str.size()) {
+    p = str.find(sub, p);
+    if (p == std::string::npos) break;
+    ++res;
+    p += sub.size();
+  }
+  return res;
+}
+
+inline int SplitOnWhitespace(const std::string& in, std::vector<std::string>* out) {
+  out->clear();
+  int i = 0;
+  int start = 0;
+  std::string cur;
+  while(i < in.size()) {
+    if (in[i] == ' ' || in[i] == '\t') {
+      if (i - start > 0)
+        out->push_back(in.substr(start, i - start));
+      start = i + 1;
+    }
+    ++i;
+  }
+  if (i > start)
+    out->push_back(in.substr(start, i - start));
+  return out->size();
+}
+
+inline void SplitCommandAndParam(const std::string& in, std::string* cmd, std::string* param) {
+  cmd->clear();
+  param->clear();
+  std::vector<std::string> x;
+  SplitOnWhitespace(in, &x);
+  if (x.size() == 0) return;
+  *cmd = x[0];
+  for (int i = 1; i < x.size(); ++i) {
+    if (i > 1) { *param += " "; }
+    *param += x[i];
+  }
+}
+
+void ProcessAndStripSGML(std::string* line, std::map<std::string, std::string>* out);
+
+// given the first character of a UTF8 block, find out how wide it is
+// see http://en.wikipedia.org/wiki/UTF-8 for more info
+inline unsigned int UTF8Len(unsigned char x) {
+  if (x < 0x80) return 1;
+  else if ((x >> 5) == 0x06) return 2;
+  else if ((x >> 4) == 0x0e) return 3;
+  else if ((x >> 3) == 0x1e) return 4;
+  else return 0;
+}
+
+#endif
diff --git a/decoder/tagger.cc b/decoder/tagger.cc
new file mode 100644
index 00000000..4dded35f
--- /dev/null
+++ b/decoder/tagger.cc
@@ -0,0 +1,112 @@
+#include "tagger.h"
+
+#include "tdict.h"
+#include "hg_io.h"
+#include "filelib.h"
+#include "hg.h"
+#include "wordid.h"
+#include "sentence_metadata.h"
+
+using namespace std;
+
+// This is a really simple linear chain tagger.
+// You specify a tagset, and it hypothesizes that each word in the
+// input can be tagged with any member of the tagset.
+// The are a couple sample features implemented in ff_tagger.h/cc
+// One thing to note, that while CRFs typically define the label
+// sequence as corresponding to the hidden states in a trellis,
+// in our model the labels are on edges, but mathematically
+// they are identical.
+//
+// Things to do if you want to make this a "real" tagger:
+// - support dictionaries (for each word, limit the tags considered)
+// - add latent variables - this is really easy to do
+
+static void ReadTagset(const string& file, vector<WordID>* tags) {
+  ReadFile rf(file);
+  istream& in(*rf.stream());
+  while(in) {
+    string tag;
+    in >> tag;
+    if (tag.empty()) continue;
+    tags->push_back(TD::Convert(tag));
+  }
+  cerr << "Read " << tags->size() << " labels (tags) from " << file << endl;
+}
+
+struct TaggerImpl {
+  TaggerImpl(const boost::program_options::variables_map& conf) :
+      kXCAT(TD::Convert("X")*-1),
+      kNULL(TD::Convert("<eps>")),
+      kBINARY(new TRule("[X] ||| [X,1] [X,2] ||| [1] [2]")),
+      kGOAL_RULE(new TRule("[Goal] ||| [X,1] ||| [1]")) {
+    if (conf.count("tagger_tagset") == 0) {
+      cerr << "Tagger requires --tagger_tagset FILE!\n";
+      exit(1);
+    }
+    ReadTagset(conf["tagger_tagset"].as<string>(), &tagset_);
+  }
+
+  void BuildTrellis(const vector<WordID>& seq, Hypergraph* forest) {
+    int prev_node_id = -1;
+    for (int i = 0; i < seq.size(); ++i) {
+      const WordID& src = seq[i];
+      const int new_node_id = forest->AddNode(kXCAT)->id_;
+      for (int k = 0; k < tagset_.size(); ++k) {
+        TRulePtr rule(TRule::CreateLexicalRule(src, tagset_[k]));
+        Hypergraph::Edge* edge = forest->AddEdge(rule, Hypergraph::TailNodeVector());
+        edge->i_ = i;
+        edge->j_ = i+1;
+        edge->prev_i_ = i;    // we set these for FastLinearIntersect
+        edge->prev_j_ = i+1;  //      "      "            "
+        forest->ConnectEdgeToHeadNode(edge->id_, new_node_id);
+      }
+      if (prev_node_id >= 0) {
+        const int comb_node_id = forest->AddNode(kXCAT)->id_;
+        Hypergraph::TailNodeVector tail(2, prev_node_id);
+        tail[1] = new_node_id;
+        Hypergraph::Edge* edge = forest->AddEdge(kBINARY, tail);
+        edge->i_ = 0;
+        edge->j_ = i+1;
+        forest->ConnectEdgeToHeadNode(edge->id_, comb_node_id);
+        prev_node_id = comb_node_id;
+      } else {
+        prev_node_id = new_node_id;
+      }
+    }
+    Hypergraph::TailNodeVector tail(1, forest->nodes_.size() - 1);
+    Hypergraph::Node* goal = forest->AddNode(TD::Convert("Goal")*-1);
+    Hypergraph::Edge* hg_edge = forest->AddEdge(kGOAL_RULE, tail);
+    forest->ConnectEdgeToHeadNode(hg_edge, goal);
+  }
+
+ private:
+  vector<WordID> tagset_;
+  const WordID kXCAT;
+  const WordID kNULL;
+  const TRulePtr kBINARY;
+  const TRulePtr kGOAL_RULE;
+};
+
+Tagger::Tagger(const boost::program_options::variables_map& conf) :
+ pimpl_(new TaggerImpl(conf)) {}
+
+
+bool Tagger::TranslateImpl(const string& input,
+                       SentenceMetadata* smeta,
+                       const vector<double>& weights,
+                       Hypergraph* forest) {
+  Lattice lattice;
+  LatticeTools::ConvertTextToLattice(input, &lattice);
+  smeta->SetSourceLength(lattice.size());
+  vector<WordID> sequence(lattice.size());
+  for (int i = 0; i < lattice.size(); ++i) {
+    assert(lattice[i].size() == 1);
+    sequence[i] = lattice[i][0].label;
+  }
+  pimpl_->BuildTrellis(sequence, forest);
+  forest->Reweight(weights);
+  forest->is_linear_chain_ = true;
+  return true;
+}
+
diff --git a/decoder/tagger.h b/decoder/tagger.h
new file mode 100644
index 00000000..9ac820d9
--- /dev/null
+++ b/decoder/tagger.h
@@ -0,0 +1,17 @@
+#ifndef _TAGGER_H_
+#define _TAGGER_H_
+
+#include "translator.h"
+
+struct TaggerImpl;
+struct Tagger : public Translator {
+  Tagger(const boost::program_options::variables_map& conf);
+  bool TranslateImpl(const std::string& input,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* forest);
+ private:
+  boost::shared_ptr<TaggerImpl> pimpl_;
+};
+
+#endif
diff --git a/decoder/tdict.cc b/decoder/tdict.cc
new file mode 100644
index 00000000..c00d20b8
--- /dev/null
+++ b/decoder/tdict.cc
@@ -0,0 +1,49 @@
+#include "Ngram.h"
+#include "dict.h"
+#include "tdict.h"
+#include "Vocab.h"
+
+using namespace std;
+
+Vocab* TD::dict_ = new Vocab;
+
+static const string empty;
+static const string space = " ";
+
+WordID TD::Convert(const std::string& s) {
+  return dict_->addWord((VocabString)s.c_str());
+}
+
+const char* TD::Convert(const WordID& w) {
+  return dict_->getWord((VocabIndex)w);
+}
+
+void TD::GetWordIDs(const std::vector<std::string>& strings, std::vector<WordID>* ids) {
+  ids->clear();
+  for (vector<string>::const_iterator i = strings.begin(); i != strings.end(); ++i)
+    ids->push_back(TD::Convert(*i));
+}
+
+std::string TD::GetString(const std::vector<WordID>& str) {
+  string res;
+  for (vector<WordID>::const_iterator i = str.begin(); i != str.end(); ++i)
+    res += (i == str.begin() ? empty : space) + TD::Convert(*i);
+  return res;
+}
+
+void TD::ConvertSentence(const std::string& sent, std::vector<WordID>* ids) {
+  string s = sent;
+  int last = 0;
+  ids->clear();
+  for (int i=0; i < s.size(); ++i)
+    if (s[i] == 32 || s[i] == '\t') {
+      s[i]=0;
+      if (last != i) {
+        ids->push_back(Convert(&s[last]));
+      }
+      last = i + 1;
+    }
+  if (last != s.size())
+    ids->push_back(Convert(&s[last]));
+}
+
diff --git a/decoder/tdict.h b/decoder/tdict.h
new file mode 100644
index 00000000..31f66367
--- /dev/null
+++ b/decoder/tdict.h
@@ -0,0 +1,30 @@
+#ifndef _TDICT_H_
+#define _TDICT_H_
+
+#include <string>
+#include <vector>
+#include "wordid.h"
+
+class Vocab;
+
+struct TD {
+  static Vocab* dict_;
+  static void ConvertSentence(const std::string& sent, std::vector<WordID>* ids);
+  static void GetWordIDs(const std::vector<std::string>& strings, std::vector<WordID>* ids);
+  static std::string GetString(const std::vector<WordID>& str);
+  static int AppendString(const WordID& w, int pos, int bufsize, char* buffer) {
+    const char* word = TD::Convert(w);
+    const char* const end_buf = buffer + bufsize;
+    char* dest = buffer + pos;
+    while(dest < end_buf && *word) {
+      *dest = *word;
+      ++dest;
+      ++word;
+    }
+    return (dest - buffer);
+  }
+  static WordID Convert(const std::string& s);
+  static const char* Convert(const WordID& w);
+};
+
+#endif
diff --git a/decoder/test_data/dummy.3gram.lm b/decoder/test_data/dummy.3gram.lm
new file mode 100644
index 00000000..ae665284
--- /dev/null
+++ b/decoder/test_data/dummy.3gram.lm
@@ -0,0 +1,2645 @@
+
+\data\
+ngram 1=490
+ngram 2=1023
+ngram 3=1119
+
+\1-grams:
+-2.761928	!	-0.06284945
+-1.91683	"	-0.03559465
+-2.761928	'	-0.06057167
+-2.159868	(	-0.07742823
+-2.159868	)	-0.05637721
+-1.292106	,	-0.04497077
+-3.062958	-	-0.06247065
+-1.429489	.	-0.08555528
+-2.761928	12	-0.06473851
+-3.062958	17	-0.06586801
+-2.585837	2000	-0.05520994
+-3.062958	2002	-0.06360606
+-3.062958	2006	-0.0497812
+-3.062958	2008	-0.06322792
+-3.062958	2009	-0.0497812
+-3.062958	200–400	-0.06549184
+-3.062958	224	-0.06586801
+-1.91683	</s>
+-99	<s>	-0.0457003
+-2.761928	?	-0.05751594
+-1.720535	a	-0.05548429
+-2.460898	about	-0.05211611
+-3.062958	acquiesced	-0.05942829
+-3.062958	actually	-0.04349266
+-3.062958	addition	-0.05980976
+-3.062958	admit	-0.06095213
+-3.062958	affected	-0.04071253
+-2.761928	against	-0.06549184
+-3.062958	aging	-0.06586801
+-3.062958	ago	-0.04349266
+-3.062958	ahead	-0.06586801
+-2.761928	al	-0.06284945
+-2.761928	all	-0.0590465
+-3.062958	all-around	-0.06586801
+-3.062958	along	-0.04071253
+-2.761928	also	-0.06322792
+-2.761928	always	-0.06436136
+-2.363988	an	-0.06436135
+-3.062958	analysis	-0.06473851
+-1.631594	and	0.006203346
+-3.062958	anti-divine	-0.06586801
+-3.062958	any	-0.06549184
+-3.062958	approach	-0.05789908
+-3.062958	archive	-0.04071253
+-3.062958	are	-0.05789908
+-2.761928	arkive	-0.06549184
+-2.585837	article	-0.0228177
+-2.21786	as	-0.09020901
+-3.062958	asked	-0.06398387
+-2.585837	at	-0.03145044
+-2.761928	attention	-0.02612664
+-3.062958	available	-0.04349266
+-3.062958	average	-0.04349266
+-3.062958	away	-0.06322792
+-3.062958	ayers	-0.05597997
+-3.062958	b	-0.04349266
+-3.062958	back-and-forth	-0.06586801
+-3.062958	bailie	-0.0497812
+-2.761928	be	-0.06511534
+-3.062958	because	-0.06586801
+-2.460898	been	-0.06322791
+-3.062958	before	-0.04349266
+-2.761928	begin	-0.05520995
+-3.062958	being	-0.06586801
+-2.585837	between	-0.1350269
+-2.460898	bias	-0.04111077
+-3.062958	biased	-0.06511534
+-3.062958	biblical	-0.06586801
+-3.062958	bill	-0.06586801
+-3.062958	blade	-0.06436136
+-3.062958	blood	-0.04349266
+-3.062958	bob	-0.06549184
+-3.062958	book	-0.06436136
+-2.159868	briffa	-0.06804922
+-2.761928	briffa's	-0.06284945
+-2.021565	but	-0.01525023
+-2.21786	by	-0.07600738
+-2.761928	ca	-0.2166343
+-2.761928	can	-0.06473851
+-3.062958	case	-0.06511534
+-3.062958	cast	-0.06473851
+-3.062958	catch	-0.06511534
+-3.062958	caught	-0.06511534
+-3.062958	caveats	-0.06322792
+-3.062958	centennial-scale	-0.06549184
+-3.062958	cf	-0.0497812
+-3.062958	change	-0.06209152
+-3.062958	changing	-0.06360606
+-3.062958	characterizes	-0.06586801
+-3.062958	checked	-0.06586801
+-2.159868	chronology	-0.02240231
+-3.062958	church	-0.06398387
+-3.062958	cocaine	-0.06398387
+-3.062958	collection	-0.06586801
+-3.062958	combination	-0.06209152
+-3.062958	combine	-0.04071253
+-3.062958	combined	-0.06209152
+-3.062958	comment	-0.06360606
+-3.062958	commentary	-0.06322792
+-3.062958	commenter	-0.06586801
+-3.062958	comments	-0.06586801
+-3.062958	compared	-0.05789908
+-3.062958	concerned	-0.06473851
+-3.062958	concrete	-0.06095213
+-3.062958	connection	-0.06209152
+-2.761928	conservatives	-0.06360606
+-3.062958	considered	-0.06095213
+-3.062958	consists	-0.04349266
+-3.062958	constructing	-0.05789908
+-2.761928	control	-0.03991493
+-2.585837	cores	-0.0236473
+-3.062958	corridor	-0.06473851
+-2.761928	crack	-0.06436136
+-3.062958	crossroads	-0.0497812
+-2.460898	cru	-0.1318786
+-3.062958	darkness	-0.05597997
+-2.108715	data	-0.06845023
+-2.761928	day	-0.05674864
+-2.761928	days	-0.04939082
+-3.062958	debt	-0.04349266
+-3.062958	decline	-0.06095213
+-3.062958	deep	-0.06549184
+-3.062958	deeper	-0.06586801
+-3.062958	delete	-0.05789908
+-3.062958	derived	-0.06511534
+-3.062958	described	-0.05942829
+-2.761928	did	-0.06095213
+-2.761928	difference	-0.04860901
+-2.761928	different	-0.06247065
+-2.761928	divergence	-0.2166343
+-2.761928	do	-0.05559513
+-3.062958	does	-0.06247065
+-3.062958	doing	-0.06586801
+-3.062958	don't	-0.06586801
+-3.062958	done	-0.06586801
+-3.062958	doubt	-0.06360606
+-3.062958	down	-0.05789908
+-3.062958	due	-0.06473851
+-3.062958	earlier	-0.06019088
+-3.062958	editors	-0.06511534
+-3.062958	energy	-0.04349266
+-3.062958	enormous	-0.06586801
+-2.761928	et	-0.2166343
+-3.062958	even	-0.06586801
+-3.062958	every	-0.06586801
+-3.062958	exactly	-0.06360606
+-3.062958	exception	-0.05789908
+-3.062958	excluding	-0.06549184
+-3.062958	expect	-0.06511534
+-3.062958	extension	-0.05597997
+-3.062958	factors	-0.04349266
+-3.062958	fantasy	-0.06436136
+-3.062958	far	-0.06511534
+-2.585837	few	-0.1590744
+-2.585837	finally	-0.06511533
+-3.062958	first	-0.04349266
+-3.062958	flesh	-0.05597997
+-3.062958	following:	-0.06095213
+-3.062958	follows:	-0.06095213
+-2.284806	for	-0.06171204
+-3.062958	forests	-0.0497812
+-2.585837	from	-0.05713245
+-3.062958	fully	-0.06586801
+-2.585837	further	-0.06511533
+-3.062958	furthermore	-0.04349266
+-3.062958	future	-0.0497812
+-3.062958	generating	-0.06586801
+-2.761928	get	-0.191855
+-3.062958	ghastly	-0.06586801
+-3.062958	ghostwritten	-0.06360606
+-3.062958	gil	-0.06586801
+-3.062958	given	-0.04071253
+-3.062958	going	-0.05789908
+-3.062958	got	-0.06436136
+-2.761928	great	-0.2166343
+-3.062958	growing	-0.0497812
+-3.062958	grows	-0.06511534
+-2.363988	had	-0.1033177
+-2.585837	hantemirov	-0.09654189
+-2.761928	happening	-0.06436136
+-3.062958	happens	-0.06549184
+-3.062958	hard	-0.05789908
+-3.062958	hardly	-0.06473851
+-2.460898	has	-0.03063563
+-3.062958	hate	-0.05789908
+-2.284806	have	-0.08108715
+-3.062958	haven't	-0.06586801
+-2.363988	he	-0.112982
+-3.062958	here	-0.06586801
+-3.062958	highly	-0.06586801
+-2.761928	him	-0.05751594
+-2.585837	his	-0.06511533
+-3.062958	how	-0.06586801
+-2.761928	however	-0.1946352
+-3.062958	hs	-0.06586801
+-3.062958	humanity	-0.06511534
+-2.108715	i	-0.05980975
+-3.062958	i'd	-0.06586801
+-3.062958	i've	-0.06586801
+-2.761928	idea	-0.02612664
+-2.761928	if	-0.03670979
+-3.062958	illusion	-0.05597997
+-3.062958	immense	-0.06586801
+-3.062958	impact	-0.06322792
+-3.062958	important	-0.06586801
+-1.807685	in	-0.04419087
+-3.062958	included	-0.06209152
+-2.761928	including	-0.0165447
+-3.062958	indeed	-0.06511534
+-3.062958	individual	-0.06511534
+-3.062958	information	-0.06511534
+-3.062958	inhomogeneities	-0.04349266
+-3.062958	initial	-0.06549184
+-2.761928	instead	-0.2109523
+-3.062958	interannual	-0.06549184
+-2.761928	into	-0.03991493
+-3.062958	introduced	-0.06360606
+-1.91683	is	-0.001109093
+-2.062958	it	-0.06621437
+-2.460898	it's	-0.06019088
+-3.062958	its	-0.06586801
+-2.761928	journal	-0.06209152
+-3.062958	jurisdiction	-0.0497812
+-2.460898	just	-0.05520994
+-3.062958	kaufman	-0.06549184
+-3.062958	keeps	-0.06586801
+-2.761928	khadyta	-0.2166343
+-2.460898	know	-0.1105378
+-3.062958	larch	-0.06586801
+-2.761928	larches	-0.04743365
+-3.062958	large-scale	-0.06095213
+-2.761928	like	-0.06511534
+-3.062958	limited	-0.06586801
+-3.062958	living	-0.06549184
+-3.062958	longest	-0.05597997
+-3.062958	looking	-0.06549184
+-3.062958	looks	-0.06586801
+-3.062958	love	-0.05789908
+-3.062958	made	-0.06095213
+-2.761928	mag	-0.2143704
+-3.062958	magnitude	-0.05980976
+-3.062958	magnus	-0.0497812
+-3.062958	makes	-0.04071253
+-3.062958	many	-0.06586801
+-3.062958	may	-0.06586801
+-3.062958	mean	-0.06322792
+-3.062958	measured	-0.06360606
+-2.761928	measurement	-0.213992
+-2.460898	method	-0.03711172
+-3.062958	methodology	-0.06586801
+-3.062958	mind	-0.06511534
+-3.062958	mix	-0.06586801
+-2.585837	more	-0.05636447
+-3.062958	morning	-0.06284945
+-2.585837	most	-0.0647385
+-2.761928	much	-0.06473851
+-3.062958	multi-parters	-0.04349266
+-3.062958	multiproxy	-0.06586801
+-3.062958	mundane	-0.06511534
+-2.585837	my	-0.1598284
+-3.062958	national	-0.06586801
+-3.062958	naughtiness	-0.0497812
+-3.062958	nettle	-0.04349266
+-3.062958	never	-0.06586801
+-3.062958	next	-0.04349266
+-3.062958	no	-0.06586801
+-3.062958	non-robustness	-0.06586801
+-3.062958	northern	-0.06586801
+-2.062958	not	-0.0712041
+-3.062958	noted	-0.06586801
+-3.062958	noticed	-0.06095213
+-3.062958	notwithstanding	-0.06473851
+-3.062958	now	-0.04349266
+-2.761928	obama	-0.03791448
+-3.062958	observed	-0.06586801
+-1.832509	of	-0.04850956
+-2.761928	old	-0.06436136
+-2.585837	older	-0.1053004
+-3.062958	oldie	-0.04349266
+-2.159868	on	-0.09226183
+-2.585837	one	-0.04900008
+-3.062958	online	-0.0497812
+-3.062958	only	-0.06586801
+-3.062958	or	-0.06586801
+-3.062958	originated	-0.06209152
+-3.062958	osborn	-0.05597997
+-3.062958	out	-0.06322792
+-3.062958	outright	-0.06586801
+-3.062958	own	-0.06586801
+-3.062958	paleoclimatologists	-0.05597997
+-3.062958	passage	-0.06284945
+-3.062958	passing	-0.05597997
+-3.062958	path	-0.06095213
+-3.062958	patterns	-0.05942829
+-3.062958	paul	-0.06436136
+-3.062958	people	-0.06095213
+-2.363988	perhaps	-0.06259563
+-2.761928	phil	-0.2166343
+-3.062958	picked	-0.06511534
+-3.062958	piece	-0.06360606
+-3.062958	place	-0.0497812
+-3.062958	placed	-0.06586801
+-3.062958	play	-0.06322792
+-3.062958	point	-0.06095213
+-3.062958	policy	-0.06322792
+-2.585837	politics	-0.02571439
+-2.363988	population	-0.1001791
+-3.062958	position	-0.06095213
+-3.062958	possible	-0.05597997
+-2.761928	potential	-0.06436136
+-3.062958	power	-0.05789908
+-3.062958	powers	-0.05597997
+-3.062958	precipitous	-0.06586801
+-3.062958	precisely	-0.04071253
+-3.062958	predictable	-0.06586801
+-3.062958	presented	-0.06019088
+-3.062958	preserve	-0.06586801
+-3.062958	previous	-0.06549184
+-3.062958	principalities	-0.05980976
+-3.062958	principles	-0.05942829
+-3.062958	prior	-0.06511534
+-3.062958	probable	-0.06095213
+-2.761928	problem	-0.2120946
+-3.062958	projected	-0.06549184
+-3.062958	properly	-0.06586801
+-3.062958	prove	-0.06586801
+-3.062958	provide	-0.04071253
+-3.062958	provided	-0.05789908
+-3.062958	provocative	-0.06586801
+-3.062958	published	-0.05942829
+-3.062958	push	-0.06511534
+-2.585837	rcs	-0.06133225
+-3.062958	react	-0.05789908
+-3.062958	read	-0.06247065
+-2.761928	readers	-0.06398387
+-3.062958	reading	-0.04349266
+-3.062958	real	-0.06322792
+-3.062958	really	-0.06586801
+-3.062958	realm	-0.05980976
+-2.761928	reason	-0.06360606
+-3.062958	recent	-0.06511534
+-2.761928	recently	-0.1946352
+-3.062958	reconstruction	-0.0497812
+-3.062958	refusal	-0.05942829
+-3.062958	refused	-0.05789908
+-3.062958	related	-0.05789908
+-3.062958	relevant	-0.04349266
+-3.062958	relied	-0.06322792
+-3.062958	religion	-0.05597997
+-3.062958	remained	-0.06586801
+-3.062958	remarked	-0.06095213
+-3.062958	reposting	-0.06473851
+-3.062958	requiring	-0.06322792
+-3.062958	response	-0.05789908
+-3.062958	resulting	-0.06322792
+-3.062958	rev	-0.0497812
+-2.460898	right	-0.04821757
+-3.062958	ring	-0.06586801
+-3.062958	ring-width	-0.06511534
+-2.761928	river	-0.1946352
+-3.062958	said	-0.06436136
+-3.062958	same	-0.06473851
+-3.062958	sample	-0.06586801
+-3.062958	sat	-0.05942829
+-2.460898	schweingruber	-0.09101291
+-3.062958	schweingruber's	-0.06549184
+-2.585837	science	-0.1568045
+-3.062958	script	-0.06322792
+-2.585837	see	-0.1112577
+-3.062958	seized	-0.04071253
+-2.761928	selected	-0.04664831
+-2.585837	selection	-0.1491516
+-3.062958	sensitive	-0.06511534
+-3.062958	sensitivity	-0.06095213
+-2.585837	series	-0.1314228
+-3.062958	set	-0.05942829
+-3.062958	several	-0.06549184
+-3.062958	shadow	-0.06586801
+-2.761928	shadows	-0.04309659
+-2.585837	shiyatov	-0.06360605
+-3.062958	should	-0.06247065
+-3.062958	similar	-0.06473851
+-3.062958	similarly	-0.06586801
+-3.062958	since	-0.06019088
+-3.062958	size	-0.05597997
+-3.062958	skimmed	-0.06019088
+-2.761928	slowly	-0.04270015
+-3.062958	small	-0.06586801
+-3.062958	so	-0.06549184
+-3.062958	some	-0.06549184
+-3.062958	someone	-0.06586801
+-3.062958	start	-0.06549184
+-3.062958	staunchly	-0.06586801
+-3.062958	struggling	-0.06549184
+-3.062958	studies	-0.06095213
+-2.761928	study	-0.02612664
+-3.062958	stumbled	-0.06586801
+-2.585837	subfossil	-0.06171205
+-3.062958	subsequent	-0.06549184
+-3.062958	subset	-0.05942829
+-3.062958	success	-0.0497812
+-3.062958	supplement	-0.0497812
+-3.062958	supplemented	-0.06360606
+-3.062958	surface	-0.04349266
+-3.062958	take	-0.06436136
+-3.062958	taken	-0.05789908
+-2.761928	taymir	-0.06247065
+-3.062958	temperature	-0.04349266
+-3.062958	tendency	-0.05789908
+-3.062958	terms	-0.05980976
+-3.062958	than	-0.04071253
+-1.91683	that	-0.06692892
+-1.243414	the	-0.08813193
+-3.062958	their	-0.06511534
+-2.761928	themselves	-0.04111078
+-3.062958	there's	-0.06586801
+-2.460898	these	-0.05942829
+-2.460898	they	-0.06398387
+-2.761928	things	-0.06057167
+-3.062958	think	-0.06549184
+-3.062958	thinking	-0.06586801
+-1.858838	this	-0.08175352
+-2.761928	those	-0.06057167
+-3.062958	thought	-0.0497812
+-3.062958	thousand	-0.04349266
+-3.062958	through	-0.04071253
+-2.761928	time	-0.0326698
+-1.720535	to	-0.07930601
+-2.761928	today	-0.04821758
+-3.062958	took	-0.04071253
+-3.062958	towards	-0.06511534
+-2.761928	trans	-0.06549184
+-2.460898	trees	-0.04704115
+-2.761928	trouble	-0.213234
+-3.062958	true	-0.04349266
+-3.062958	trying	-0.05789908
+-2.761928	two	-0.2166343
+-3.062958	unarchived	-0.0497812
+-3.062958	under	-0.06549184
+-3.062958	unintentional	-0.06473851
+-3.062958	unrepresentativeness	-0.05980976
+-3.062958	until	-0.06549184
+-3.062958	unveiled:	-0.06586801
+-2.761928	up	-0.03185729
+-3.062958	upon	-0.06019088
+-2.761928	use	-0.2109523
+-2.363988	used	-0.0545155
+-2.761928	using	-0.02323271
+-3.062958	usual	-0.06586801
+-3.062958	valid	-0.06549184
+-2.761928	variability	-0.03911585
+-2.761928	versions	-0.04428373
+-2.761928	very	-0.06549184
+-3.062958	violence	-0.06586801
+-3.062958	virtually	-0.06586801
+-3.062958	virtue	-0.05980976
+-3.062958	voted	-0.06398387
+-3.062958	warn	-0.06549184
+-3.062958	warnings	-0.04349266
+-2.363988	was	-0.06171205
+-3.062958	way	-0.06549184
+-3.062958	we	-0.06549184
+-3.062958	well	-0.06398387
+-2.284806	were	-0.07866543
+-2.21786	what	-0.02364731
+-3.062958	what's	-0.06549184
+-2.585837	when	-0.06057167
+-2.585837	where	-0.05597997
+-2.460898	which	-0.0403139
+-2.585837	while	-0.03951557
+-3.062958	whose	-0.06586801
+-3.062958	why	-0.06586801
+-3.062958	widths	-0.05597997
+-2.761928	will	-0.06322792
+-3.062958	wise	-0.06549184
+-2.021565	with	-0.08912028
+-3.062958	within	-0.06549184
+-3.062958	without	-0.06586801
+-3.062958	worth	-0.06586801
+-2.460898	would	-0.1303614
+-3.062958	wright's	-0.06586801
+-3.062958	wrote	-0.04071253
+-2.159868	yamal	-0.0719028
+-2.761928	year	-0.04270015
+-3.062958	years	-0.06549184
+-3.062958	yes	-0.04349266
+-3.062958	yesterday	-0.06473851
+-3.062958	yet	-0.04349266
+-3.062958	you	-0.06511534
+-2.761928	your	-0.06511534
+
+\2-grams:
+-1.15037	! as	-0.004049858
+-1.15037	! instead	0.2044696
+-1.995468	" (	-0.005168174
+-1.995468	" -	0.05332709
+-1.995468	" </s>
+-1.995468	" as	-0.004049858
+-1.995468	" concrete	0.05332709
+-1.995468	" corridor	0.05332709
+-1.249819	" divergence	0.1451325
+-1.995468	" further	0.008061528
+-1.995468	" i'd	0.05332709
+-1.995468	" success	0.05332709
+-1.995468	" that	-0.008505944
+-1.995468	" the	-0.007702977
+-1.995468	" used	-0.0004517734
+-1.15037	' </s>
+-1.15037	' yes	0.05332709
+-1.75243	( and	-0.01063527
+-1.75243	( in	0.006514465
+-1.006781	( mag	0.1451325
+-1.75243	( or	0.05332709
+-1.75243	( phil	0.2044696
+-1.75243	( which	0.00272119
+-1.75243	( while	0.008061528
+-1.006781	) ,	-0.002172916
+-1.75243	) </s>
+-1.75243	) acquiesced	0.05332709
+-1.75243	) and	-0.002266581
+-1.75243	) had	-0.0004517734
+-1.75243	) things	0.01894335
+-1.75243	) took	0.05332709
+-2.620192	, 2008	0.05332709
+-2.620192	, 224	0.05332709
+-2.620192	, a	-0.01011507
+-2.620192	, all	0.01894335
+-1.955229	, and	-0.006035992
+-2.620192	, as	0.0389223
+-2.620192	, bob	0.05332709
+-2.620192	, briffa	-0.005168174
+-0.8166095	, but	0.05114232
+-2.620192	, cf	0.05332709
+-2.620192	, cru	0.00272119
+-2.620192	, delete	0.05332709
+-2.620192	, for	-0.002554279
+-2.620192	, from	0.008061528
+-2.620192	, he	-0.0004517734
+-2.620192	, his	0.008061528
+-1.955229	, i	0.008061524
+-2.620192	, if	0.01894335
+-2.620192	, including	0.01894335
+-2.620192	, is	-0.008505944
+-1.874543	, it	-0.0004517762
+-1.874543	, it's	0.01894334
+-2.620192	, kaufman	0.05332709
+-2.620192	, most	0.008061528
+-2.620192	, notwithstanding	0.05332709
+-2.620192	, of	0.007685009
+-2.620192	, on	-0.005168174
+-2.620192	, perhaps	0.04797027
+-2.620192	, requiring	0.05332709
+-2.620192	, since	0.05332709
+-1.955229	, the	0.02331641
+-1.955229	, this	0.01715922
+-2.620192	, until	0.05332709
+-2.620192	, using	0.01894335
+-1.874543	, when	0.03010483
+-2.620192	, where	0.008061528
+-1.874543	, which	0.01894334
+-2.620192	, while	0.008061528
+-2.620192	, yamal	-0.005168174
+-0.8493397	- not	-0.006728992
+-2.482808	. "	-0.008505944
+-2.482808	. '	0.01894335
+-2.482808	. (	-0.005168174
+-2.482808	. )	-0.005168174
+-0.6792259	. </s>
+-1.737159	. a	0.003078613
+-2.482808	. actually	0.05332709
+-2.482808	. and	-0.01063527
+-2.482808	. as	-0.004049858
+-1.737159	. briffa	0.03257156
+-2.482808	. but	-0.007295175
+-2.482808	. changing	0.05332709
+-2.482808	. first	0.05332709
+-2.482808	. furthermore	0.05332709
+-1.737159	. however	0.1451325
+-2.482808	. i	-0.006035987
+-2.482808	. in	-0.009490006
+-2.482808	. it	0.0164606
+-2.482808	. perhaps	0.04797027
+-2.482808	. science	0.1193421
+-2.482808	. several	0.05332709
+-2.482808	. the	-0.008591395
+-1.737159	. these	0.01894334
+-1.737159	. this	0.0130633
+-2.482808	. violence	0.05332709
+-2.482808	. what	-0.004049858
+-2.482808	. what's	0.05332709
+-2.482808	. while	0.008061528
+-2.482808	. with	0.05785327
+-2.482808	. wright's	0.05332709
+-1.15037	12 cores	0.008061528
+-1.15037	12 picked	0.05332709
+-0.8493397	17 ring-width	0.05332709
+-1.326461	2000 and	-0.01063527
+-1.326461	2000 may	0.05332709
+-1.326461	2000 presented	0.05332709
+-0.8493397	2002 as	-0.004049858
+-0.8493397	2006 .	-0.0114856
+-0.8493397	2008 )	-0.005168174
+-0.8493397	2009 .	0.08907277
+-0.8493397	200–400 year	0.01894335
+-0.8493397	224 individual	0.05332709
+-1.995468	<s> '	0.01894335
+-1.995468	<s> as	0.0389223
+-1.995468	<s> briffa's	0.01894335
+-1.995468	<s> but	-0.007295175
+-1.995468	<s> i	-0.006035987
+-1.995468	<s> if	0.01894335
+-1.995468	<s> in	-0.009490006
+-1.995468	<s> next	0.05332709
+-1.249819	<s> perhaps	0.06234263
+-1.249819	<s> the	0.0223057
+-1.995468	<s> this	-0.009059753
+-1.995468	<s> what	-0.004049858
+-1.15037	? "	-0.008505944
+-1.15037	? i	-0.006035987
+-2.191762	a "	0.01222976
+-2.191762	a case	0.05332709
+-2.191762	a comment	0.05332709
+-2.191762	a commenter	0.05332709
+-2.191762	a different	0.01894335
+-1.5268	a few	0.109396
+-2.191762	a generating	0.05332709
+-2.191762	a great	0.2044696
+-2.191762	a mean	0.05332709
+-2.191762	a prior	0.05332709
+-2.191762	a provocative	0.05332709
+-2.191762	a rcs	0.008061528
+-2.191762	a science	0.008061528
+-2.191762	a shadow	0.05332709
+-2.191762	a similar	0.05332709
+-2.191762	a small	0.05332709
+-2.191762	a surface	0.05332709
+-2.191762	a thousand	0.05332709
+-2.191762	a time	0.01894335
+-2.191762	a valid	0.05332709
+-1.4514	about a	-0.01011507
+-1.4514	about my	0.008061528
+-1.4514	about not	-0.006728992
+-1.4514	about potential	0.01894335
+-0.8493397	acquiesced in	-0.009490006
+-0.8493397	actually ,	-0.01187418
+-0.8493397	addition of	-0.009287588
+-0.8493397	admit that	0.04168737
+-0.8493397	affected the	-0.01198488
+-1.15037	against flesh	0.05332709
+-1.15037	against inhomogeneities	0.05332709
+-0.8493397	aging patterns	0.05332709
+-0.8493397	ago ,	-0.008075343
+-0.8493397	ahead you	0.05332709
+-1.15037	al (	-0.005168174
+-1.15037	al 2009	0.05332709
+-1.15037	all of	-0.009287588
+-1.15037	all those	0.01894335
+-0.8493397	all-around naughtiness	0.05332709
+-0.8493397	along the	-0.01198488
+-1.15037	also has	0.00272119
+-1.15037	also know	0.08231446
+-1.15037	always been	0.00272119
+-1.15037	always worth	0.05332709
+-1.54831	an exception	0.05332709
+-1.54831	an extension	0.05332709
+-1.54831	an immense	0.05332709
+-1.54831	an important	0.05332709
+-1.54831	an unintentional	0.05332709
+-0.8493397	analysis has	0.00272119
+-2.280704	and ,	-0.007080218
+-2.280704	and all-around	0.05332709
+-2.280704	and blood	0.05332709
+-2.280704	and briffa	-0.005168174
+-2.280704	and even	0.05332709
+-2.280704	and got	0.05332709
+-2.280704	and hantemirov	0.09388901
+-2.280704	and he	0.06152429
+-2.280704	and i've	0.05332709
+-2.280704	and it	-0.006728992
+-2.280704	and most	0.008061528
+-2.280704	and outright	0.05332709
+-2.280704	and perhaps	-0.0004517734
+-2.280704	and politics	0.008061528
+-2.280704	and potential	0.01894335
+-2.280704	and principalities	0.05332709
+-2.280704	and sat	0.05332709
+-2.280704	and science	0.1193421
+-1.615741	and shiyatov	0.05332708
+-2.280704	and temperature	0.05332709
+-2.280704	and that	-0.008505944
+-1.615741	and the	-0.005814605
+-2.280704	and they	0.00272119
+-0.8493397	anti-divine powers	0.05332709
+-0.8493397	any journal	0.01894335
+-0.8493397	approach to	-0.01011507
+-0.8493397	archive the	-0.01198488
+-0.8493397	are to	-0.01011507
+-1.15037	arkive down	0.05332709
+-1.15037	arkive under	0.05332709
+-1.326461	article ,	-0.007080218
+-1.326461	article .	-0.004888296
+-1.326461	article on	-0.005168174
+-1.694438	as a	-0.01011507
+-0.9487888	as ca	0.1451325
+-1.694438	as compared	0.05332709
+-1.694438	as follows:	0.05332709
+-1.694438	as it	0.0164606
+-1.694438	as noted	0.05332709
+-0.8493397	asked for	-0.002554279
+-1.326461	at a	-0.01011507
+-1.326461	at precisely	0.05332709
+-1.326461	at the	-0.01198488
+-1.15037	attention ,	0.05896524
+-1.15037	attention .	-0.0114856
+-0.8493397	available ,	-0.008075343
+-0.8493397	average ,	-0.01187418
+-0.8493397	away )	0.03209379
+-0.8493397	ayers and	-0.01063527
+-0.8493397	b ,	-0.01187418
+-0.8493397	back-and-forth yesterday	0.05332709
+-0.8493397	bailie .	-0.0114856
+-1.15037	be happening	0.01894335
+-1.15037	be included	0.05332709
+-0.8493397	because so	0.05332709
+-1.4514	been an	-0.0004517734
+-1.4514	been concerned	0.05332709
+-1.4514	been done	0.05332709
+-1.4514	been projected	0.05332709
+-0.8493397	before ,	-0.01187418
+-1.15037	begin in	-0.009490006
+-1.15037	begin with	-0.007295175
+-0.8493397	being true	0.05332709
+-1.326461	between ring	0.05332709
+-0.580812	between the	-0.06704012
+-1.4514	bias ,	-0.007080218
+-1.4514	bias introduced	0.05332709
+-1.4514	bias towards	0.05332709
+-1.4514	bias would	0.08231446
+-0.8493397	biased selection	0.1193421
+-0.8493397	biblical passage	0.05332709
+-0.8493397	bill ayers	0.05332709
+-0.8493397	blade was	-0.0004517734
+-0.8493397	blood ,	0.05896524
+-0.8493397	bob ?	0.01894335
+-0.8493397	book was	-0.0004517734
+-1.087467	briffa 2000	0.05332708
+-1.75243	briffa 2006	0.05332709
+-1.75243	briffa asked	0.05332709
+-1.75243	briffa et	0.2044696
+-1.75243	briffa to	-0.01011507
+-1.75243	briffa used	-0.0004517734
+-1.15037	briffa's own	0.05332709
+-1.15037	briffa's yamal	-0.005168174
+-1.890732	but ,	-0.01187418
+-1.890732	but anti-divine	0.05332709
+-1.890732	but because	0.05332709
+-1.890732	but between	0.1193421
+-1.890732	but given	0.05332709
+-1.145083	but it	-0.0004517762
+-1.890732	but it's	0.00272119
+-1.890732	but the	-0.01198488
+-1.890732	but this	0.009005655
+-1.890732	but to	0.002916232
+-1.694438	by bill	0.05332709
+-1.694438	by gil	0.05332709
+-1.694438	by hantemirov	0.09388901
+-1.694438	by how	0.05332709
+-1.694438	by magnus	0.05332709
+-0.9487888	by the	-0.01105098
+-0.4047208	ca readers	0.05332709
+-1.15037	can combine	0.05332709
+-1.15037	can see	0.1193421
+-0.8493397	case where	0.008061528
+-0.8493397	cast these	0.00272119
+-0.8493397	catch my	0.1193421
+-0.8493397	caught my	0.1193421
+-0.8493397	caveats on	-0.005168174
+-0.8493397	centennial-scale variability	0.01894335
+-0.8493397	cf .	-0.0114856
+-0.8493397	change with	-0.007295175
+-0.8493397	changing what	-0.004049858
+-0.8493397	characterizes northern	0.05332709
+-0.8493397	checked earlier	0.05332709
+-1.75243	chronology ,	-0.01187418
+-1.75243	chronology also	0.01894335
+-1.75243	chronology briffa	-0.005168174
+-1.75243	chronology has	0.00272119
+-1.75243	chronology in	-0.009490006
+-1.75243	chronology method	0.00272119
+-1.75243	chronology was	-0.0004517734
+-1.75243	chronology with	-0.007295175
+-0.8493397	church for	-0.002554279
+-0.8493397	cocaine for	-0.002554279
+-0.8493397	collection does	0.05332709
+-0.8493397	combination with	0.05785327
+-0.8493397	combine the	-0.01198488
+-0.8493397	combined with	0.05785327
+-0.8493397	comment by	-0.004049858
+-0.8493397	commentary on	0.03209379
+-0.8493397	commenter remarked	0.05332709
+-0.8493397	comments catch	0.05332709
+-0.8493397	compared to	0.02102831
+-0.8493397	concerned about	0.00272119
+-0.8493397	concrete "	-0.008505944
+-0.8493397	connection with	-0.007295175
+-1.15037	conservatives said	0.05332709
+-1.15037	conservatives were	-0.002554279
+-0.8493397	considered "	-0.008505944
+-0.8493397	consists ,	-0.01187418
+-0.8493397	constructing a	-0.01011507
+-1.15037	control !	0.01894335
+-1.15037	control the	-0.01198488
+-1.326461	cores ,	-0.008075343
+-1.326461	cores .	-0.004888296
+-1.326461	cores were	0.04819728
+-0.8493397	corridor method	0.00272119
+-1.15037	crack about	0.00272119
+-1.15037	crack cocaine	0.05332709
+-0.8493397	crossroads .	-0.0114856
+-0.7057508	cru population	0.07636014
+-1.4514	cru selection	0.008061528
+-1.4514	cru staunchly	0.05332709
+-0.8493397	darkness and	-0.01063527
+-1.803582	data (	-0.005168174
+-1.057933	data .	-0.0100497
+-1.803582	data policy	0.05332709
+-1.803582	data remained	0.05332709
+-1.803582	data set	0.05332709
+-1.803582	data used	0.04797027
+-1.803582	data was	-0.0004517734
+-1.803582	data were	-0.002554279
+-1.15037	day politics	0.008061528
+-1.15037	day to	-0.01011507
+-1.15037	days .	0.08907277
+-1.15037	days ago	0.05332709
+-0.8493397	debt ,	-0.007080218
+-0.8493397	decline is	-0.008505944
+-0.8493397	deep into	0.01894335
+-0.8493397	deeper principles	0.05332709
+-0.8493397	delete a	0.0001907796
+-0.8493397	derived from	0.008061528
+-0.8493397	described in	-0.009490006
+-1.15037	did not	-0.006728992
+-1.15037	did they	0.00272119
+-1.15037	difference .	0.08907277
+-1.15037	difference between	0.1193421
+-1.15037	different aging	0.05332709
+-1.15037	different data	-0.006035987
+-0.4047208	divergence problem	0.1451325
+-1.15037	do and	-0.002266581
+-1.15037	do indeed	0.05332709
+-0.8493397	does not	0.0164606
+-0.8493397	doing exactly	0.05332709
+-0.8493397	don't really	0.05332709
+-0.8493397	done without	0.05332709
+-0.8493397	doubt what	-0.004049858
+-0.8493397	down to	-0.01011507
+-0.8493397	due just	0.00272119
+-0.8493397	earlier this	-0.009059753
+-0.8493397	editors finally	0.008061528
+-0.8493397	energy ,	0.05896524
+-0.8493397	enormous hs	0.05332709
+-0.4047208	et al	0.05332709
+-0.8493397	even probable	0.05332709
+-0.8493397	every subsequent	0.05332709
+-0.8493397	exactly what	-0.004049858
+-0.8493397	exception to	0.02102831
+-0.8493397	excluding khadyta	0.2044696
+-0.8493397	expect from	0.008061528
+-0.8493397	extension and	-0.01063527
+-0.8493397	factors ,	0.05896524
+-0.8493397	fantasy had	-0.0004517734
+-0.8493397	far more	0.008061528
+-1.326461	few at	0.008061528
+-0.580812	few days	0.05332709
+-1.326461	finally available	0.05332709
+-1.326461	finally placed	0.05332709
+-1.326461	finally seized	0.05332709
+-0.8493397	first ,	-0.01187418
+-0.8493397	flesh and	-0.01063527
+-0.8493397	following: </s>
+-0.8493397	follows: </s>
+-1.627491	for all	0.01894335
+-1.627491	for an	-0.0004517734
+-1.627491	for excluding	0.05332709
+-1.627491	for him	0.01894335
+-1.627491	for paleoclimatologists	0.05332709
+-1.627491	for we	0.05332709
+-0.8493397	forests .	-0.004888296
+-1.326461	from 200–400	0.05332709
+-1.326461	from a	-0.01011507
+-1.326461	from someone	0.05332709
+-0.8493397	fully thinking	0.05332709
+-1.326461	further ahead	0.05332709
+-1.326461	further along	0.05332709
+-1.326461	further away	0.05332709
+-0.8493397	furthermore ,	-0.007080218
+-0.8493397	future .	-0.0114856
+-0.8493397	generating script	0.05332709
+-0.4047208	get the	-0.06704012
+-0.8493397	ghastly tendency	0.05332709
+-0.8493397	ghostwritten by	-0.004049858
+-0.8493397	gil bailie	0.05332709
+-0.8493397	given the	-0.01198488
+-0.8493397	going to	-0.01011507
+-0.8493397	got used	0.04797027
+-0.4047208	great idea	0.05332709
+-0.8493397	growing .	0.08907277
+-0.8493397	grows more	0.008061528
+-0.8026608	had a	-0.007295178
+-1.54831	had been	0.00272119
+-1.54831	had in	-0.009490006
+-1.54831	had jurisdiction	0.05332709
+-0.6614985	hantemirov and	-0.5914098
+-1.15037	happening deep	0.05332709
+-1.15037	happening right	0.00272119
+-0.8493397	happens today	0.01894335
+-0.8493397	hard to	-0.01011507
+-0.8493397	hardly know	0.00272119
+-1.4514	has a	-0.01011507
+-1.4514	has always	0.01894335
+-1.4514	has only	0.05332709
+-1.4514	has the	-0.01198488
+-0.8493397	hate to	-0.01011507
+-1.627491	have an	-0.0004517734
+-0.881842	have been	0.01894334
+-1.627491	have relied	0.05332709
+-1.627491	have similarly	0.05332709
+-1.627491	have the	-0.01198488
+-0.8493397	haven't read	0.05332709
+-0.8026608	he is	-0.004049861
+-1.54831	he made	0.05332709
+-1.54831	he would	0.00272119
+-1.54831	he wrote	0.05332709
+-0.8493397	here prove	0.05332709
+-0.8493397	highly possible	0.05332709
+-1.15037	him hate	0.05332709
+-1.15037	him to	0.002916232
+-1.326461	his comments	0.05332709
+-1.326461	his initial	0.05332709
+-1.326461	his precipitous	0.05332709
+-0.8493397	how their	0.05332709
+-0.4047208	however ,	-0.01082908
+-0.8493397	hs blade	0.05332709
+-0.8493397	humanity at	0.008061528
+-1.803582	i can	0.01894335
+-1.803582	i checked	0.05332709
+-1.803582	i had	0.06152429
+-1.803582	i hardly	0.05332709
+-1.803582	i haven't	0.05332709
+-1.803582	i know	0.00272119
+-1.803582	i noticed	0.05332709
+-1.803582	i skimmed	0.05332709
+-1.803582	i stumbled	0.05332709
+-0.8493397	i'd love	0.05332709
+-0.8493397	i've provided	0.05332709
+-1.15037	idea ,	-0.01187418
+-1.15037	idea .	-0.0114856
+-1.15037	if it	-0.006728992
+-1.15037	if the	-0.01198488
+-0.8493397	illusion and	-0.01063527
+-0.8493397	immense energy	0.05332709
+-0.8493397	impact on	-0.005168174
+-0.8493397	important impact	0.05332709
+-1.358963	in a	-0.007295178
+-2.104612	in any	0.05332709
+-2.104612	in briffa	0.02412629
+-2.104612	in briffa's	0.01894335
+-2.104612	in combination	0.05332709
+-2.104612	in connection	0.05332709
+-2.104612	in hantemirov	0.09388901
+-2.104612	in mind	0.05332709
+-2.104612	in one	0.008061528
+-2.104612	in passing	0.05332709
+-2.104612	in response	0.05332709
+-2.104612	in rev	0.05332709
+-2.104612	in terms	0.05332709
+-1.358963	in the	-0.007650165
+-2.104612	in this	-0.009059753
+-2.104612	in virtually	0.05332709
+-0.8493397	included with	0.05785327
+-1.15037	including ,	-0.01187418
+-1.15037	including the	-0.007702977
+-0.8493397	indeed see	0.1193421
+-0.8493397	individual series	0.008061528
+-0.8493397	information finally	0.008061528
+-0.8493397	inhomogeneities ,	0.05896524
+-0.8493397	initial use	0.2044696
+-0.4047208	instead of	0.01149127
+-0.8493397	interannual variability	0.01894335
+-1.15037	into him	0.01894335
+-1.15037	into the	-0.01198488
+-0.8493397	introduced by	-0.004049858
+-1.995468	is ,	-0.007080218
+-1.995468	is always	0.01894335
+-1.995468	is considered	0.05332709
+-1.995468	is derived	0.05332709
+-1.995468	is doing	0.05332709
+-1.995468	is happening	0.01894335
+-1.995468	is highly	0.05332709
+-1.995468	is measured	0.05332709
+-1.995468	is no	0.05332709
+-1.995468	is not	-0.006728992
+-1.995468	is related	0.05332709
+-1.995468	is that	-0.008505944
+-1.995468	is the	-0.01198488
+-1.995468	is within	0.05332709
+-1.84934	it grows	0.05332709
+-1.84934	it has	0.00272119
+-1.184377	it is	0.0004524188
+-1.84934	it just	0.00272119
+-1.84934	it looks	0.05332709
+-1.84934	it originated	0.05332709
+-1.84934	it was	-0.0004517734
+-1.84934	it yet	0.05332709
+-1.4514	it's like	0.01894335
+-1.4514	it's much	0.01894335
+-1.4514	it's not	-0.006728992
+-1.4514	it's very	0.01894335
+-0.8493397	its enormous	0.05332709
+-1.15037	journal (	-0.005168174
+-1.15037	journal article	0.008061528
+-0.8493397	jurisdiction .	-0.004888296
+-1.4514	just between	0.008061528
+-1.4514	just keeps	0.05332709
+-1.4514	just one	0.008061528
+-1.4514	just to	0.02102831
+-0.8493397	kaufman et	0.2044696
+-0.8493397	keeps growing	0.05332709
+-0.4047208	khadyta river	0.1451325
+-1.4514	know !	0.01894335
+-0.7057508	know ,	-0.007021053
+-1.4514	know where	0.008061528
+-0.8493397	larch sample	0.05332709
+-1.15037	larches .	0.08907277
+-1.15037	larches were	0.04819728
+-0.8493397	large-scale "	0.01222976
+-1.15037	like crack	0.01894335
+-1.15037	like trying	0.05332709
+-0.8493397	limited size	0.05332709
+-0.8493397	living larches	0.01894335
+-0.8493397	longest and	-0.01063527
+-0.8493397	looking up	0.01894335
+-0.8493397	looks relevant	0.05332709
+-0.8493397	love to	-0.01011507
+-0.8493397	made that	-0.008505944
+-0.4047208	mag )	0.002721187
+-0.8493397	magnitude of	-0.009287588
+-0.8493397	magnus .	-0.0114856
+-0.8493397	makes the	-0.01198488
+-0.8493397	many multiproxy	0.05332709
+-0.8493397	may well	0.05332709
+-0.8493397	mean chronology	-0.005168174
+-0.8493397	measured by	0.0389223
+-0.4047208	measurement data	0.0009555696
+-1.4514	method "	-0.008505944
+-1.4514	method .	-0.004888296
+-1.4514	method that	-0.008505944
+-1.4514	method which	0.00272119
+-0.8493397	methodology warn	0.05332709
+-0.8493397	mind when	0.008061528
+-0.8493397	mix religion	0.05332709
+-1.326461	more "	-0.008505944
+-1.326461	more it	0.0164606
+-1.326461	more slowly	0.01894335
+-0.8493397	morning i	-0.006035987
+-1.326461	most recent	0.05332709
+-1.326461	most recently	0.2044696
+-1.326461	most sensitive	0.05332709
+-1.15037	much further	0.008061528
+-1.15037	much illusion	0.05332709
+-0.8493397	multi-parters ,	-0.01187418
+-0.8493397	multiproxy studies	0.05332709
+-0.8493397	mundane politics	0.008061528
+-0.580812	my attention	0.05332709
+-1.326461	my ghastly	0.05332709
+-0.8493397	national debt	0.05332709
+-0.8493397	naughtiness .	-0.0114856
+-0.8493397	nettle ,	-0.01187418
+-0.8493397	never properly	0.05332709
+-0.8493397	next ,	-0.008075343
+-0.8493397	no doubt	0.05332709
+-0.8493397	non-robustness observed	0.05332709
+-0.8493397	northern forests	0.05332709
+-1.84934	not be	0.01894335
+-1.84934	not due	0.05332709
+-1.84934	not going	0.05332709
+-1.184377	not have	0.07243546
+-1.84934	not just	0.00272119
+-1.84934	not preserve	0.05332709
+-1.84934	not struggling	0.05332709
+-1.84934	not using	0.01894335
+-0.8493397	noted before	0.05332709
+-0.8493397	noticed that	0.04168737
+-0.8493397	notwithstanding these	0.00272119
+-0.8493397	now ,	0.05896524
+-1.15037	obama ,	-0.007080218
+-1.15037	obama is	-0.008505944
+-0.8493397	observed here	0.05332709
+-2.079789	of 17	0.05332709
+-2.079789	of a	-0.01011507
+-2.079789	of being	0.05332709
+-2.079789	of commentary	0.05332709
+-2.079789	of darkness	0.05332709
+-2.079789	of deeper	0.05332709
+-2.079789	of his	0.008061528
+-2.079789	of interannual	0.05332709
+-2.079789	of mundane	0.05332709
+-2.079789	of old	0.01894335
+-1.33414	of older	0.03455187
+-2.079789	of reposting	0.05332709
+-2.079789	of subfossil	0.008061528
+-1.33414	of the	-0.06704012
+-2.079789	of this	-0.009059753
+-1.15037	old living	0.05332709
+-1.15037	old trees	0.00272119
+-0.6614985	older trees	0.03579502
+-0.8493397	oldie ,	-0.008075343
+-1.006781	on a	-0.007295178
+-1.75243	on average	0.05332709
+-1.75243	on many	0.05332709
+-1.75243	on rcs	0.008061528
+-1.75243	on the	-0.007702977
+-1.006781	on this	-0.005168174
+-1.326461	one .	-0.0114856
+-1.326461	one approach	0.05332709
+-1.326461	one oldie	0.05332709
+-0.8493397	online .	-0.0114856
+-0.8493397	only taken	0.05332709
+-0.8493397	or real	0.05332709
+-0.8493397	originated with	-0.007295175
+-0.8493397	osborn and	-0.01063527
+-0.8493397	out (	-0.005168174
+-0.8493397	outright fantasy	0.05332709
+-0.8493397	own caveats	0.05332709
+-0.8493397	paleoclimatologists and	-0.01063527
+-0.8493397	passage i	-0.006035987
+-0.8493397	passing and	-0.01063527
+-0.8493397	path "	-0.008505944
+-0.8493397	patterns in	0.006514465
+-0.8493397	paul had	-0.0004517734
+-0.8493397	people that	-0.008505944
+-0.8833473	perhaps the	-0.01011507
+-1.54831	perhaps there's	0.05332709
+-1.54831	perhaps they	0.00272119
+-0.4047208	phil trans	0.05332709
+-0.8493397	picked cores	0.008061528
+-0.8493397	piece by	-0.004049858
+-0.8493397	place .	-0.0114856
+-0.8493397	placed online	0.05332709
+-0.8493397	play on	0.03209379
+-0.8493397	point that	-0.008505944
+-0.8493397	policy )	-0.005168174
+-1.326461	politics ,	-0.01187418
+-1.326461	politics .	-0.004888296
+-1.326461	politics are	0.05332709
+-0.8026608	population .	-0.0100497
+-1.54831	population as	-0.004049858
+-1.54831	population consists	0.05332709
+-1.54831	population instead	0.2044696
+-0.8493397	position that	0.04168737
+-0.8493397	possible and	-0.01063527
+-1.15037	potential bias	0.00272119
+-1.15037	potential unrepresentativeness	0.05332709
+-0.8493397	power to	-0.01011507
+-0.8493397	powers and	-0.01063527
+-0.8493397	precipitous decline	0.05332709
+-0.8493397	precisely the	-0.007702977
+-0.8493397	predictable factors	0.05332709
+-0.8493397	presented this	0.009005655
+-0.8493397	preserve centennial-scale	0.05332709
+-0.8493397	previous journal	0.01894335
+-0.8493397	principalities of	-0.009287588
+-0.8493397	principles in	0.006514465
+-0.8493397	prior selection	0.1193421
+-0.8493397	probable that	0.04168737
+-0.4047208	problem "	-0.004049861
+-0.8493397	projected into	0.01894335
+-0.8493397	properly published	0.05332709
+-0.8493397	prove out	0.05332709
+-0.8493397	provide the	-0.01198488
+-0.8493397	provided a	-0.01011507
+-0.8493397	provocative thought	0.05332709
+-0.8493397	published in	-0.009490006
+-0.8493397	push at	0.008061528
+-1.326461	rcs chronology	-0.005168174
+-1.326461	rcs method	0.00272119
+-1.326461	rcs methodology	0.05332709
+-0.8493397	react to	0.002916232
+-0.8493397	read it	-0.006728992
+-1.15037	readers also	0.01894335
+-1.15037	readers know	0.08231446
+-0.8493397	reading ,	-0.01187418
+-0.8493397	real )	-0.005168174
+-0.8493397	really react	0.05332709
+-0.8493397	realm of	-0.009287588
+-1.15037	reason for	-0.002554279
+-1.15037	reason why	0.05332709
+-0.8493397	recent one	0.008061528
+-0.4047208	recently ,	-0.01082908
+-0.8493397	reconstruction .	-0.0114856
+-0.8493397	refusal in	-0.009490006
+-0.8493397	refused to	-0.01011507
+-0.8493397	related to	-0.01011507
+-0.8493397	relevant ,	-0.008075343
+-0.8493397	relied on	0.03209379
+-0.8493397	religion and	-0.01063527
+-0.8493397	remained unarchived	0.05332709
+-0.8493397	remarked that	-0.008505944
+-0.8493397	reposting just	0.00272119
+-0.8493397	requiring briffa	-0.005168174
+-0.8493397	response to	0.02102831
+-0.8493397	resulting yamal	0.02412629
+-0.8493397	rev .	-0.0114856
+-1.4514	right .	-0.0114856
+-1.4514	right now	0.05332709
+-1.4514	right place	0.05332709
+-1.4514	right time	0.01894335
+-0.8493397	ring widths	0.05332709
+-0.8493397	ring-width series	0.1193421
+-0.4047208	river ,	-0.01082908
+-0.8493397	said he	-0.0004517734
+-0.8493397	same bias	0.00272119
+-0.8493397	sample should	0.05332709
+-0.8493397	sat in	-0.009490006
+-1.4514	schweingruber data	-0.006035987
+-0.7864373	schweingruber population	0.09172077
+-0.8493397	schweingruber's khadyta	0.2044696
+-0.580812	science (	-0.02724335
+-1.326461	science article	0.008061528
+-0.8493397	script )	0.03209379
+-1.326461	see ,	-0.008075343
+-0.580812	see the	-0.01105098
+-0.8493397	seized the	-0.01198488
+-1.15037	selected .	-0.004888296
+-1.15037	selected on	0.03209379
+-1.326461	selection is	-0.008505944
+-0.580812	selection of	0.01149127
+-0.8493397	sensitive series	0.1193421
+-0.8493397	sensitivity is	-0.008505944
+-0.580812	series ,	-0.01082908
+-1.326461	series of	-0.009287588
+-0.8493397	set in	-0.009490006
+-0.8493397	several things	0.01894335
+-0.8493397	shadow play	0.05332709
+-1.15037	shadows .	-0.0114856
+-1.15037	shadows of	-0.009287588
+-1.326461	shiyatov 2002	0.05332709
+-1.326461	shiyatov themselves	0.01894335
+-1.326461	shiyatov would	0.08231446
+-0.8493397	should not	-0.006728992
+-0.8493397	similar schweingruber	0.00272119
+-0.8493397	similarly affected	0.05332709
+-0.8493397	since this	-0.009059753
+-0.8493397	size and	-0.01063527
+-0.8493397	skimmed this	-0.009059753
+-1.15037	slowly ,	-0.01187418
+-1.15037	slowly get	0.2044696
+-0.8493397	small push	0.05332709
+-0.8493397	so much	0.01894335
+-0.8493397	some reason	0.01894335
+-0.8493397	someone whose	0.05332709
+-0.8493397	start today	0.01894335
+-0.8493397	staunchly refused	0.05332709
+-0.8493397	struggling against	0.01894335
+-0.8493397	studies that	-0.008505944
+-1.15037	study ,	-0.01187418
+-1.15037	study .	0.08907277
+-0.8493397	stumbled upon	0.05332709
+-1.326461	subfossil collection	0.05332709
+-1.326461	subfossil data	0.02685598
+-1.326461	subfossil larches	0.01894335
+-0.8493397	subsequent study	0.01894335
+-0.8493397	subset in	-0.009490006
+-0.8493397	success .	-0.0114856
+-0.8493397	supplement .	0.08907277
+-0.8493397	supplemented by	0.0389223
+-0.8493397	surface ,	-0.008075343
+-0.8493397	take an	-0.0004517734
+-0.8493397	taken a	0.0001907796
+-1.15037	taymir data	-0.006035987
+-1.15037	taymir supplement	0.05332709
+-0.8493397	temperature ,	0.05896524
+-0.8493397	tendency to	-0.01011507
+-0.8493397	terms of	-0.009287588
+-0.8493397	than the	-0.008591395
+-1.995468	that "	-0.008505944
+-1.995468	that cast	0.05332709
+-1.995468	that characterizes	0.05332709
+-1.995468	that have	-0.002554279
+-1.995468	that he	0.06152429
+-1.995468	that his	0.008061528
+-0.9275748	that the	0.03271748
+-1.995468	that they	0.00272119
+-1.995468	that voted	0.05332709
+-1.995468	that way	0.05332709
+-1.995468	that wise	0.05332709
+-2.668884	the "	-0.008505944
+-1.923235	the 12	0.05332709
+-2.668884	the addition	0.05332709
+-1.923235	the arkive	0.05332709
+-2.668884	the back-and-forth	0.05332709
+-2.668884	the biased	0.05332709
+-2.668884	the biblical	0.05332709
+-2.668884	the chronology	-0.005168174
+-1.923235	the conservatives	0.05332709
+-2.668884	the crossroads	0.05332709
+-2.003921	the cru	0.0632299
+-2.668884	the data	0.02685598
+-2.668884	the day	0.01894335
+-2.668884	the difference	0.01894335
+-2.668884	the far	0.05332709
+-2.668884	the following:	0.05332709
+-2.668884	the further	0.008061528
+-2.668884	the future	0.05332709
+-2.668884	the information	0.05332709
+-2.668884	the large-scale	0.05332709
+-2.668884	the longest	0.05332709
+-2.668884	the magnitude	0.05332709
+-2.668884	the measurement	0.2044696
+-2.668884	the more	0.008061528
+-2.668884	the most	0.008061528
+-2.668884	the multi-parters	0.05332709
+-2.668884	the national	0.05332709
+-2.668884	the nettle	0.05332709
+-2.668884	the non-robustness	0.05332709
+-2.668884	the path	0.05332709
+-2.668884	the people	0.05332709
+-2.668884	the phil	0.2044696
+-2.668884	the point	0.05332709
+-2.668884	the position	0.05332709
+-2.668884	the previous	0.05332709
+-2.668884	the rcs	0.008061528
+-2.668884	the realm	0.05332709
+-2.668884	the resulting	0.05332709
+-1.923235	the right	0.01894334
+-2.668884	the same	0.05332709
+-2.003921	the schweingruber	-0.5245172
+-2.668884	the shadows	0.01894335
+-2.668884	the subfossil	0.008061528
+-1.923235	the taymir	0.05332709
+-1.923235	the trouble	0.1451325
+-1.923235	the two	0.1451325
+-2.668884	the use	0.2044696
+-2.668884	the usual	0.05332709
+-2.668884	the very	0.01894335
+-2.668884	the virtue	0.05332709
+-1.120574	the yamal	0.02719982
+-0.8493397	their cores	0.008061528
+-1.15037	themselves ,	-0.01187418
+-1.15037	themselves were	-0.002554279
+-0.8493397	there's some	0.05332709
+-1.4514	these data	-0.006035987
+-1.4514	these shadows	0.01894335
+-1.4514	these warnings	0.05332709
+-1.4514	these were	-0.002554279
+-1.4514	they can	0.01894335
+-1.4514	they don't	0.05332709
+-1.4514	they expect	0.05332709
+-1.4514	they themselves	0.01894335
+-1.15037	things caught	0.05332709
+-1.15037	things that	-0.008505944
+-0.8493397	think up	0.01894335
+-0.8493397	thinking through	0.05332709
+-2.05346	this analysis	0.05332709
+-2.05346	this article	0.008061528
+-2.05346	this bias	0.00272119
+-1.307811	this chronology	0.002721187
+-2.05346	this difference	0.01894335
+-1.307811	this is	-0.004049861
+-2.05346	this method	0.00272119
+-2.05346	this morning	0.05332709
+-2.05346	this piece	0.05332709
+-2.05346	this refusal	0.05332709
+-2.05346	this study	0.01894335
+-2.05346	this subset	0.05332709
+-2.05346	this will	0.01894335
+-2.05346	this year	0.01894335
+-1.15037	those "	-0.008505944
+-1.15037	those years	0.05332709
+-0.8493397	thought .	-0.0114856
+-0.8493397	thousand ,	0.05896524
+-0.8493397	through the	-0.01198488
+-1.15037	time ,	-0.008075343
+-1.15037	time and	-0.002266581
+-2.191762	to about	0.00272119
+-2.191762	to admit	0.05332709
+-2.191762	to archive	0.05332709
+-1.446113	to begin	0.05332709
+-2.191762	to change	0.05332709
+-2.191762	to constructing	0.05332709
+-2.191762	to control	0.01894335
+-2.191762	to day	0.01894335
+-2.191762	to different	0.01894335
+-2.191762	to get	0.2044696
+-2.191762	to mix	0.05332709
+-2.191762	to provide	0.05332709
+-2.191762	to start	0.05332709
+-1.123869	to the	-0.005761562
+-2.191762	to think	0.05332709
+-2.191762	to those	0.01894335
+-1.446113	to what	0.005001867
+-1.15037	today .	-0.0114856
+-1.15037	today would	0.00272119
+-0.8493397	took the	-0.01198488
+-0.8493397	towards older	0.09388901
+-1.15037	trans b	0.05332709
+-1.15037	trans editors	0.05332709
+-1.4514	trees .	-0.0114856
+-1.4514	trees an	-0.0004517734
+-1.4514	trees described	0.05332709
+-1.4514	trees than	0.05332709
+-0.4047208	trouble with	-0.03998877
+-0.8493397	true ,	-0.01187418
+-0.8493397	trying to	-0.01011507
+-0.4047208	two versions	0.05332709
+-0.8493397	unarchived .	-0.004888296
+-0.8493397	under control	0.01894335
+-0.8493397	unintentional bias	0.00272119
+-0.8493397	unrepresentativeness of	0.007685009
+-0.8493397	until recently	0.2044696
+-0.8493397	unveiled: humanity	0.05332709
+-1.15037	up a	-0.01011507
+-1.15037	up the	-0.01198488
+-0.8493397	upon this	-0.009059753
+-0.4047208	use of	-0.005627823
+-1.54831	used by	-0.004049858
+-0.8833473	used in	0.01371272
+-1.54831	used the	-0.01198488
+-1.15037	using .	0.08907277
+-1.15037	using the	-0.008591395
+-0.8493397	usual predictable	0.05332709
+-0.8493397	valid reason	0.01894335
+-1.15037	variability .	-0.004888296
+-1.15037	variability and	-0.01063527
+-1.15037	versions .	0.08907277
+-1.15037	versions is	-0.008505944
+-1.15037	very hard	0.05332709
+-1.15037	very limited	0.05332709
+-0.8493397	violence unveiled:	0.05332709
+-0.8493397	virtually every	0.05332709
+-0.8493397	virtue of	-0.009287588
+-0.8493397	voted for	-0.002554279
+-0.8493397	warn against	0.01894335
+-0.8493397	warnings ,	-0.01187418
+-1.54831	was finally	0.008061528
+-1.54831	was ghostwritten	0.05332709
+-1.54831	was like	0.01894335
+-1.54831	was never	0.05332709
+-1.54831	was used	0.04797027
+-0.8493397	way slowly	0.01894335
+-0.8493397	we do	0.01894335
+-0.8493397	well have	0.04819728
+-1.627491	were not	-0.006728992
+-1.627491	were right	0.00272119
+-0.881842	were selected	0.05332709
+-1.627491	were supplemented	0.05332709
+-1.627491	were the	-0.01198488
+-1.694438	what a	-0.01011507
+-1.694438	what did	0.01894335
+-1.694438	what happens	0.05332709
+-1.694438	what is	-0.008505944
+-1.694438	what paul	0.05332709
+-1.694438	what the	-0.007702977
+-1.694438	what will	0.01894335
+-0.8493397	what's your	0.01894335
+-1.326461	when combined	0.05332709
+-1.326461	when he	-0.0004517734
+-1.326461	when i	-0.006035987
+-1.326461	where it's	0.00272119
+-1.326461	where sensitivity	0.05332709
+-1.326461	where to	0.002916232
+-1.4514	which ,	-0.01187418
+-1.4514	which did	0.01894335
+-1.4514	which had	0.06152429
+-1.4514	which makes	0.05332709
+-1.326461	while including	0.01894335
+-1.326461	while looking	0.05332709
+-1.326461	while the	0.02129733
+-0.8493397	whose book	0.05332709
+-0.8493397	why schweingruber's	0.05332709
+-0.8493397	widths and	-0.01063527
+-1.15037	will be	0.01894335
+-1.15037	will have	-0.002554279
+-0.8493397	wise crack	0.01894335
+-1.890732	with .	-0.004888296
+-1.890732	with a	-0.01011507
+-1.890732	with briffa	0.02412629
+-1.890732	with its	0.05332709
+-1.145083	with obama	0.05332709
+-1.890732	with osborn	0.05332709
+-0.8228394	with the	0.02898683
+-0.8493397	within your	0.01894335
+-0.8493397	without fully	0.05332709
+-0.8493397	worth reading	0.05332709
+-1.4514	would do	0.01894335
+-0.7057508	would not	-0.04287655
+-1.4514	would take	0.05332709
+-0.8493397	wright's church	0.05332709
+-0.8493397	wrote the	-0.01198488
+-1.087467	yamal chronology	0.01075652
+-1.75243	yamal data	-0.006035987
+-1.75243	yamal larch	0.05332709
+-1.75243	yamal measurement	0.2044696
+-1.75243	yamal reconstruction	0.05332709
+-1.75243	yamal subfossil	0.008061528
+-1.15037	year ,	-0.008075343
+-1.15037	year old	0.01894335
+-0.8493397	years ?	0.01894335
+-0.8493397	yes ,	-0.01187418
+-0.8493397	yesterday about	0.00272119
+-0.8493397	yet ,	0.05896524
+-0.8493397	you see	0.008061528
+-1.15037	your great	0.2044696
+-1.15037	your power	0.05332709
+
+\3-grams:
+-1.533073	control ! as
+-1.533073	know ! instead
+-1.533073	. " i'd
+-1.533073	? " </s>
+-1.533073	a " divergence
+-1.533073	concrete " (
+-1.533073	considered " success
+-1.533073	large-scale " divergence
+-1.533073	method " used
+-1.533073	more " concrete
+-1.533073	path " as
+-1.834103	problem " -
+-1.834103	problem " that
+-1.533073	that " the
+-1.533073	the " corridor
+-1.533073	those " further
+-1.533073	. ' </s>
+-1.533073	<s> ' yes
+-1.533073	" ( or
+-1.533073	. ( while
+-1.533073	al ( phil
+-1.533073	data ( in
+-1.533073	journal ( which
+-1.533073	out ( and
+-0.8145491	science ( mag
+-1.533073	. ) </s>
+-1.533073	2008 ) and
+-1.533073	away ) ,
+-1.834103	mag ) acquiesced
+-1.834103	mag ) took
+-1.533073	policy ) had
+-1.533073	real ) things
+-1.533073	script ) ,
+-1.834103	) , it's
+-1.834103	) , this
+-1.533073	actually , all
+-1.533073	ago , i
+-1.533073	and , when
+-1.533073	article , it
+-1.533073	attention , but
+-1.533073	available , this
+-1.533073	average , of
+-1.533073	b , 2008
+-1.533073	before , briffa
+-1.533073	bias , when
+-1.533073	blood , but
+-1.533073	but , notwithstanding
+-1.533073	chronology , 224
+-1.533073	consists , on
+-1.533073	cores , this
+-1.533073	debt , which
+-1.533073	energy , but
+-1.533073	factors , but
+-1.533073	first , a
+-1.533073	furthermore , it
+-1.834103	however , as
+-1.834103	however , using
+-1.533073	idea , bob
+-1.533073	including , most
+-1.533073	inhomogeneities , but
+-1.533073	is , it's
+-1.834103	know , the
+-1.834103	know , until
+-1.533073	multi-parters , delete
+-1.533073	nettle , requiring
+-1.533073	next , i
+-1.533073	now , but
+-1.533073	obama , which
+-1.533073	oldie , i
+-1.533073	politics , he
+-1.533073	reading , cf
+-1.834103	recently , cru
+-1.834103	recently , kaufman
+-1.533073	relevant , and
+-1.834103	river , while
+-1.834103	river , yamal
+-1.533073	see , the
+-1.834103	series , from
+-1.834103	series , where
+-1.533073	slowly , is
+-1.533073	study , including
+-1.533073	surface , and
+-1.533073	temperature , but
+-1.533073	themselves , since
+-1.533073	thousand , but
+-1.533073	time , and
+-1.533073	true , for
+-1.533073	warnings , his
+-1.533073	which , if
+-1.533073	year , the
+-1.533073	yes , perhaps
+-1.533073	yet , but
+-1.533073	" - not
+-1.533073	2006 . while
+-1.533073	2009 . </s>
+-1.533073	article . however
+-1.533073	attention . first
+-1.533073	bailie . i
+-1.533073	cf . violence
+-1.533073	cores . briffa
+-1.533073	crossroads . )
+-1.834103	data . as
+-1.834103	data . but
+-1.533073	days . </s>
+-1.533073	difference . </s>
+-1.533073	forests . however
+-1.533073	future . changing
+-1.533073	growing . </s>
+-1.533073	idea . what's
+-1.533073	jurisdiction . briffa
+-1.533073	larches . </s>
+-1.533073	magnus . actually
+-1.533073	method . this
+-1.533073	naughtiness . (
+-1.533073	one . in
+-1.533073	online . with
+-1.533073	place . '
+-1.533073	politics . this
+-1.834103	population . it
+-1.834103	population . the
+-1.533073	reconstruction . science
+-1.533073	rev . wright's
+-1.533073	right . what
+-1.533073	selected . these
+-1.533073	shadows . and
+-1.533073	study . </s>
+-1.533073	success . "
+-1.533073	supplement . </s>
+-1.533073	thought . furthermore
+-1.533073	today . several
+-1.533073	trees . perhaps
+-1.533073	unarchived . a
+-1.533073	using . </s>
+-1.533073	variability . these
+-1.533073	versions . </s>
+-1.533073	with . a
+-1.834103	the 12 cores
+-1.834103	the 12 picked
+-1.533073	of 17 ring-width
+-2.010194	briffa 2000 and
+-2.010194	briffa 2000 may
+-2.010194	briffa 2000 presented
+-1.533073	shiyatov 2002 as
+-1.533073	briffa 2006 .
+-1.533073	, 2008 )
+-1.533073	al 2009 .
+-1.533073	from 200–400 year
+-1.533073	, 224 individual
+-1.533073	bob ? i
+-1.533073	years ? "
+-1.533073	, a comment
+-1.834103	. a commenter
+-1.834103	. a few
+-1.533073	about a thousand
+-1.533073	as a shadow
+-1.533073	at a time
+-1.533073	constructing a mean
+-1.533073	delete a few
+-1.533073	from a prior
+-1.834103	had a different
+-1.834103	had a great
+-1.533073	has a "
+-1.834103	in a case
+-1.834103	in a science
+-1.533073	of a similar
+-1.834103	on a rcs
+-1.834103	on a surface
+-1.533073	provided a generating
+-1.533073	taken a few
+-1.533073	up a valid
+-1.533073	what a provocative
+-1.533073	with a small
+-1.533073	concerned about potential
+-1.533073	crack about not
+-1.533073	to about a
+-1.533073	yesterday about my
+-1.533073	) acquiesced in
+-1.533073	. actually ,
+-1.533073	the addition of
+-1.533073	to admit that
+-1.533073	similarly affected the
+-1.533073	struggling against flesh
+-1.533073	warn against inhomogeneities
+-1.533073	different aging patterns
+-1.533073	days ago ,
+-1.533073	further ahead you
+-1.834103	et al (
+-1.834103	et al 2009
+-1.533073	, all of
+-1.533073	for all those
+-1.533073	and all-around naughtiness
+-1.533073	further along the
+-1.533073	chronology also has
+-1.533073	readers also know
+-1.533073	has always been
+-1.533073	is always worth
+-1.533073	been an exception
+-1.533073	for an extension
+-1.533073	have an important
+-1.533073	take an immense
+-1.533073	trees an unintentional
+-1.533073	this analysis has
+-1.533073	( and i've
+-1.533073	) and the
+-2.010194	, and he
+-2.010194	, and that
+-2.010194	, and they
+-1.533073	. and perhaps
+-1.533073	2000 and science
+-1.533073	ayers and sat
+-1.533073	darkness and all-around
+-1.533073	do and the
+-1.533073	extension and ,
+-1.533073	flesh and blood
+-0.1249387	hantemirov and shiyatov
+-1.533073	illusion and outright
+-1.533073	longest and most
+-1.533073	osborn and briffa
+-1.533073	paleoclimatologists and got
+-1.533073	passing and it
+-1.533073	possible and even
+-1.533073	powers and principalities
+-1.533073	religion and politics
+-1.533073	size and potential
+-1.533073	time and the
+-1.533073	variability and hantemirov
+-1.533073	widths and temperature
+-1.533073	but anti-divine powers
+-1.533073	in any journal
+-1.533073	one approach to
+-1.533073	to archive the
+-1.533073	politics are to
+-1.834103	the arkive down
+-1.834103	the arkive under
+-1.533073	journal article .
+-1.533073	science article ,
+-1.533073	this article on
+-1.533073	! as it
+-1.533073	" as a
+-1.533073	, as ca
+-1.533073	. as noted
+-1.533073	2002 as follows:
+-1.533073	<s> as ca
+-1.533073	population as compared
+-1.533073	briffa asked for
+-1.533073	few at a
+-1.533073	humanity at the
+-1.533073	push at precisely
+-1.834103	my attention ,
+-1.834103	my attention .
+-1.533073	finally available ,
+-1.533073	on average ,
+-1.533073	further away )
+-1.533073	bill ayers and
+-1.533073	trans b ,
+-1.533073	the back-and-forth yesterday
+-1.533073	gil bailie .
+-1.533073	not be included
+-1.533073	will be happening
+-1.533073	but because so
+-1.533073	always been an
+-1.533073	had been projected
+-1.834103	have been concerned
+-1.834103	have been done
+-1.533073	noted before ,
+-1.834103	to begin in
+-1.834103	to begin with
+-1.533073	of being true
+-1.533073	but between the
+-1.533073	difference between the
+-1.533073	just between ring
+-1.533073	potential bias introduced
+-1.533073	same bias towards
+-1.533073	this bias would
+-1.533073	unintentional bias ,
+-1.533073	the biased selection
+-1.533073	the biblical passage
+-1.533073	by bill ayers
+-1.533073	hs blade was
+-1.533073	and blood ,
+-1.533073	, bob ?
+-1.533073	whose book was
+-1.533073	, briffa asked
+-1.834103	. briffa 2000
+-1.834103	. briffa used
+-1.533073	and briffa 2006
+-1.533073	chronology briffa et
+-1.533073	in briffa 2000
+-1.533073	requiring briffa to
+-1.533073	with briffa 2000
+-1.533073	<s> briffa's own
+-1.533073	in briffa's yamal
+-2.487315	, but ,
+-2.487315	, but anti-divine
+-2.487315	, but because
+-2.487315	, but between
+-1.467762	, but it
+-2.487315	, but the
+-2.487315	, but this
+-2.487315	, but to
+-1.533073	. but given
+-1.533073	<s> but it's
+-1.533073	comment by magnus
+-1.533073	ghostwritten by bill
+-1.533073	introduced by how
+-1.533073	measured by the
+-1.533073	piece by gil
+-1.533073	supplemented by the
+-1.533073	used by hantemirov
+-0.8145491	as ca readers
+-1.533073	i can combine
+-1.533073	they can see
+-1.533073	a case where
+-1.533073	that cast these
+-1.533073	comments catch my
+-1.533073	things caught my
+-1.533073	own caveats on
+-1.533073	preserve centennial-scale variability
+-1.533073	, cf .
+-1.533073	to change with
+-1.533073	. changing what
+-1.533073	that characterizes northern
+-1.533073	i checked earlier
+-1.533073	mean chronology ,
+-1.533073	rcs chronology method
+-1.533073	the chronology briffa
+-1.834103	this chronology also
+-1.834103	this chronology in
+-2.010194	yamal chronology has
+-2.010194	yamal chronology was
+-2.010194	yamal chronology with
+-1.533073	wright's church for
+-1.533073	crack cocaine for
+-1.533073	subfossil collection does
+-1.533073	in combination with
+-1.533073	can combine the
+-1.533073	when combined with
+-1.533073	a comment by
+-1.533073	of commentary on
+-1.533073	a commenter remarked
+-1.533073	his comments catch
+-1.533073	as compared to
+-1.533073	been concerned about
+-1.533073	" concrete "
+-1.533073	in connection with
+-1.834103	the conservatives said
+-1.834103	the conservatives were
+-1.533073	is considered "
+-1.533073	population consists ,
+-1.533073	to constructing a
+-1.533073	to control the
+-1.533073	under control !
+-1.533073	12 cores .
+-1.533073	picked cores ,
+-1.533073	their cores were
+-1.533073	" corridor method
+-1.533073	like crack cocaine
+-1.533073	wise crack about
+-1.533073	the crossroads .
+-1.533073	, cru staunchly
+-0.9906404	the cru population
+-2.010194	the cru selection
+-1.533073	of darkness and
+-1.533073	different data policy
+-1.834103	measurement data remained
+-1.834103	measurement data used
+-1.533073	schweingruber data set
+-1.533073	subfossil data .
+-1.533073	taymir data (
+-1.533073	the data .
+-1.533073	these data were
+-1.533073	yamal data was
+-1.533073	the day to
+-1.533073	to day politics
+-1.834103	few days .
+-1.834103	few days ago
+-1.533073	national debt ,
+-1.533073	precipitous decline is
+-1.533073	happening deep into
+-1.533073	of deeper principles
+-1.533073	, delete a
+-1.533073	is derived from
+-1.533073	trees described in
+-1.533073	what did they
+-1.533073	which did not
+-1.533073	the difference between
+-1.533073	this difference .
+-1.533073	a different data
+-1.533073	to different aging
+-0.8145491	" divergence problem
+-1.533073	we do indeed
+-1.533073	would do and
+-1.533073	collection does not
+-1.533073	is doing exactly
+-1.533073	they don't really
+-1.533073	been done without
+-1.533073	no doubt what
+-1.533073	arkive down to
+-1.533073	not due just
+-1.533073	checked earlier this
+-1.533073	trans editors finally
+-1.533073	immense energy ,
+-1.533073	its enormous hs
+-1.533073	briffa et al
+-1.533073	kaufman et al
+-1.533073	and even probable
+-1.533073	virtually every subsequent
+-1.533073	doing exactly what
+-1.533073	an exception to
+-1.533073	for excluding khadyta
+-1.533073	they expect from
+-1.533073	an extension and
+-1.533073	predictable factors ,
+-1.533073	outright fantasy had
+-1.533073	the far more
+-2.010194	a few at
+-0.9906404	a few days
+-1.533073	editors finally seized
+-1.533073	information finally available
+-1.533073	was finally placed
+-1.533073	. first ,
+-1.533073	against flesh and
+-1.533073	the following: </s>
+-1.533073	as follows: </s>
+-1.533073	, for we
+-1.533073	asked for an
+-1.533073	church for all
+-1.533073	cocaine for paleoclimatologists
+-1.533073	reason for excluding
+-1.533073	voted for him
+-1.533073	northern forests .
+-1.533073	, from 200–400
+-1.533073	derived from a
+-1.533073	expect from someone
+-1.533073	without fully thinking
+-1.533073	" further along
+-1.533073	much further away
+-1.533073	the further ahead
+-1.533073	. furthermore ,
+-1.533073	the future .
+-1.533073	a generating script
+-1.533073	slowly get the
+-1.533073	to get the
+-1.533073	my ghastly tendency
+-1.533073	was ghostwritten by
+-1.533073	by gil bailie
+-1.533073	but given the
+-1.533073	not going to
+-1.533073	and got used
+-1.533073	a great idea
+-1.533073	your great idea
+-1.533073	keeps growing .
+-1.533073	it grows more
+-1.533073	) had jurisdiction
+-1.533073	fantasy had been
+-1.533073	i had a
+-1.533073	paul had in
+-1.533073	which had a
+-1.533073	and hantemirov and
+-1.533073	by hantemirov and
+-1.533073	in hantemirov and
+-1.533073	be happening deep
+-1.533073	is happening right
+-1.533073	what happens today
+-1.533073	very hard to
+-1.533073	i hardly know
+-1.533073	also has a
+-1.533073	analysis has only
+-1.533073	chronology has always
+-1.533073	it has the
+-1.533073	him hate to
+-2.010194	not have been
+-2.010194	not have similarly
+-2.010194	not have the
+-1.533073	that have relied
+-1.533073	well have been
+-1.533073	will have an
+-1.533073	i haven't read
+-1.533073	, he wrote
+-1.533073	and he is
+-1.533073	said he would
+-1.533073	that he is
+-1.533073	when he made
+-1.533073	observed here prove
+-1.533073	is highly possible
+-1.533073	for him hate
+-1.533073	into him to
+-1.533073	, his initial
+-1.533073	of his comments
+-1.533073	that his precipitous
+-1.533073	by how their
+-0.8145491	. however ,
+-1.533073	enormous hs blade
+-1.533073	unveiled: humanity at
+-2.010194	, i can
+-2.010194	, i noticed
+-2.010194	, i skimmed
+-1.533073	. i haven't
+-1.533073	<s> i hardly
+-1.533073	? i know
+-1.533073	morning i had
+-1.533073	passage i stumbled
+-1.533073	when i checked
+-1.533073	" i'd love
+-1.533073	and i've provided
+-1.834103	great idea ,
+-1.834103	great idea .
+-1.533073	, if it
+-1.533073	<s> if the
+-1.533073	much illusion and
+-1.533073	an immense energy
+-1.533073	important impact on
+-1.533073	an important impact
+-1.533073	( in a
+-1.533073	. in response
+-1.533073	<s> in one
+-1.533073	acquiesced in this
+-1.533073	begin in terms
+-1.533073	chronology in passing
+-1.533073	described in hantemirov
+-1.533073	had in mind
+-1.533073	patterns in the
+-1.533073	principles in the
+-1.533073	published in any
+-1.533073	refusal in connection
+-1.533073	sat in rev
+-1.533073	set in combination
+-1.533073	subset in briffa
+-2.010194	used in a
+-2.010194	used in briffa's
+-2.010194	used in virtually
+-1.533073	be included with
+-1.533073	, including ,
+-1.533073	while including the
+-1.533073	do indeed see
+-1.533073	224 individual series
+-1.533073	the information finally
+-1.533073	against inhomogeneities ,
+-1.533073	his initial use
+-1.533073	! instead of
+-1.533073	population instead of
+-1.533073	of interannual variability
+-1.533073	deep into the
+-1.533073	projected into him
+-1.533073	bias introduced by
+-1.533073	, is considered
+-1.533073	decline is not
+-1.834103	he is always
+-1.834103	he is doing
+-2.010194	it is ,
+-2.010194	it is highly
+-2.010194	it is within
+-1.533073	obama is that
+-1.533073	selection is derived
+-1.533073	sensitivity is measured
+-1.834103	this is no
+-1.834103	this is the
+-1.533073	versions is related
+-1.533073	what is happening
+-1.834103	, it has
+-1.834103	, it originated
+-1.533073	. it is
+-1.533073	and it was
+-1.533073	as it is
+-1.834103	but it just
+-1.834103	but it looks
+-1.533073	if it grows
+-1.533073	more it is
+-1.533073	read it yet
+-1.834103	, it's like
+-1.834103	, it's very
+-1.533073	but it's not
+-1.533073	where it's much
+-1.533073	with its enormous
+-1.533073	any journal article
+-1.533073	previous journal (
+-1.533073	had jurisdiction .
+-1.533073	due just to
+-1.533073	it just keeps
+-1.533073	not just between
+-1.533073	reposting just one
+-1.533073	, kaufman et
+-1.533073	just keeps growing
+-1.533073	excluding khadyta river
+-1.533073	schweingruber's khadyta river
+-1.533073	also know ,
+-1.533073	hardly know where
+-1.533073	i know !
+-1.533073	readers know ,
+-1.533073	yamal larch sample
+-1.533073	living larches .
+-1.533073	subfossil larches were
+-1.533073	the large-scale "
+-1.533073	it's like trying
+-1.533073	was like crack
+-1.533073	very limited size
+-1.533073	old living larches
+-1.533073	the longest and
+-1.533073	while looking up
+-1.533073	it looks relevant
+-1.533073	i'd love to
+-1.533073	he made that
+-0.8145491	( mag )
+-1.533073	the magnitude of
+-1.533073	by magnus .
+-1.533073	which makes the
+-1.533073	on many multiproxy
+-1.533073	2000 may well
+-1.533073	a mean chronology
+-1.533073	is measured by
+-1.533073	the measurement data
+-1.533073	yamal measurement data
+-1.533073	chronology method that
+-1.533073	corridor method "
+-1.533073	rcs method .
+-1.533073	this method which
+-1.533073	rcs methodology warn
+-1.533073	in mind when
+-1.533073	to mix religion
+-1.533073	far more "
+-1.533073	grows more slowly
+-1.533073	the more it
+-1.533073	this morning i
+-1.533073	, most recently
+-1.533073	and most sensitive
+-1.533073	the most recent
+-1.533073	it's much further
+-1.533073	so much illusion
+-1.533073	the multi-parters ,
+-1.533073	many multiproxy studies
+-1.533073	of mundane politics
+-1.533073	about my ghastly
+-1.533073	catch my attention
+-1.533073	caught my attention
+-1.533073	the national debt
+-1.533073	all-around naughtiness .
+-1.533073	the nettle ,
+-1.533073	was never properly
+-1.533073	<s> next ,
+-1.533073	is no doubt
+-1.533073	the non-robustness observed
+-1.533073	characterizes northern forests
+-1.533073	- not just
+-1.533073	about not struggling
+-1.533073	did not preserve
+-1.533073	does not have
+-1.533073	is not due
+-1.533073	it's not going
+-1.533073	should not be
+-1.533073	were not using
+-0.8145491	would not have
+-1.533073	as noted before
+-1.533073	i noticed that
+-1.533073	, notwithstanding these
+-1.533073	right now ,
+-1.834103	with obama ,
+-1.834103	with obama is
+-1.533073	non-robustness observed here
+-1.533073	, of older
+-1.533073	addition of 17
+-1.533073	all of his
+-1.834103	instead of reposting
+-1.834103	instead of the
+-1.533073	magnitude of interannual
+-1.533073	principalities of darkness
+-1.533073	realm of mundane
+-1.834103	selection of old
+-1.834103	selection of older
+-1.533073	series of subfossil
+-1.533073	shadows of deeper
+-1.533073	terms of commentary
+-1.533073	unrepresentativeness of the
+-1.834103	use of a
+-1.834103	use of this
+-1.533073	virtue of being
+-1.533073	of old trees
+-1.533073	year old living
+-0.8145491	of older trees
+-1.533073	towards older trees
+-1.533073	one oldie ,
+-1.533073	, on average
+-1.533073	article on the
+-1.533073	caveats on rcs
+-1.533073	commentary on this
+-1.533073	impact on many
+-1.533073	play on a
+-1.533073	relied on this
+-1.533073	selected on a
+-1.533073	in one approach
+-1.533073	just one oldie
+-1.533073	recent one .
+-1.533073	placed online .
+-1.533073	has only taken
+-1.533073	( or real
+-1.533073	it originated with
+-1.533073	with osborn and
+-1.533073	prove out (
+-1.533073	and outright fantasy
+-1.533073	briffa's own caveats
+-1.533073	for paleoclimatologists and
+-1.533073	biblical passage i
+-1.533073	in passing and
+-1.533073	the path "
+-1.533073	aging patterns in
+-1.533073	what paul had
+-1.533073	the people that
+-1.533073	, perhaps the
+-1.533073	. perhaps the
+-1.834103	<s> perhaps the
+-1.834103	<s> perhaps there's
+-1.533073	and perhaps they
+-1.533073	( phil trans
+-1.533073	the phil trans
+-1.533073	12 picked cores
+-1.533073	this piece by
+-1.533073	right place .
+-1.533073	finally placed online
+-1.533073	shadow play on
+-1.533073	the point that
+-1.533073	data policy )
+-1.533073	and politics ,
+-1.533073	day politics are
+-1.533073	mundane politics .
+-1.834103	cru population .
+-1.834103	cru population consists
+-2.010194	schweingruber population .
+-2.010194	schweingruber population as
+-2.010194	schweingruber population instead
+-1.533073	the position that
+-1.533073	highly possible and
+-1.533073	about potential bias
+-1.533073	and potential unrepresentativeness
+-1.533073	your power to
+-1.533073	anti-divine powers and
+-1.533073	his precipitous decline
+-1.533073	at precisely the
+-1.533073	usual predictable factors
+-1.533073	2000 presented this
+-1.533073	not preserve centennial-scale
+-1.533073	the previous journal
+-1.533073	and principalities of
+-1.533073	deeper principles in
+-1.533073	a prior selection
+-1.533073	even probable that
+-0.8145491	divergence problem "
+-1.533073	been projected into
+-1.533073	never properly published
+-1.533073	here prove out
+-1.533073	to provide the
+-1.533073	i've provided a
+-1.533073	a provocative thought
+-1.533073	properly published in
+-1.533073	small push at
+-1.533073	a rcs chronology
+-1.533073	on rcs methodology
+-1.533073	the rcs method
+-1.533073	really react to
+-1.533073	haven't read it
+-1.834103	ca readers also
+-1.834103	ca readers know
+-1.533073	worth reading ,
+-1.533073	or real )
+-1.533073	don't really react
+-1.533073	the realm of
+-1.533073	some reason why
+-1.533073	valid reason for
+-1.533073	most recent one
+-1.533073	most recently ,
+-1.533073	until recently ,
+-1.533073	yamal reconstruction .
+-1.533073	this refusal in
+-1.533073	staunchly refused to
+-1.533073	is related to
+-1.533073	looks relevant ,
+-1.533073	have relied on
+-1.533073	mix religion and
+-1.533073	data remained unarchived
+-1.533073	commenter remarked that
+-1.533073	of reposting just
+-1.533073	, requiring briffa
+-1.533073	in response to
+-1.533073	the resulting yamal
+-1.533073	in rev .
+-1.533073	happening right now
+-1.834103	the right place
+-1.834103	the right time
+-1.533073	were right .
+-1.533073	between ring widths
+-1.533073	17 ring-width series
+-0.8145491	khadyta river ,
+-1.533073	conservatives said he
+-1.533073	the same bias
+-1.533073	larch sample should
+-1.533073	and sat in
+-1.533073	similar schweingruber data
+-0.1249387	the schweingruber population
+-1.533073	why schweingruber's khadyta
+-1.533073	. science (
+-1.533073	a science article
+-1.533073	and science (
+-1.533073	generating script )
+-1.533073	can see the
+-1.533073	indeed see the
+-1.533073	you see ,
+-1.533073	finally seized the
+-1.834103	were selected .
+-1.834103	were selected on
+-1.533073	biased selection of
+-1.533073	cru selection is
+-1.533073	prior selection of
+-1.533073	most sensitive series
+-1.533073	where sensitivity is
+-1.533073	individual series of
+-1.533073	ring-width series ,
+-1.533073	sensitive series ,
+-1.533073	data set in
+-1.533073	. several things
+-1.533073	a shadow play
+-1.533073	the shadows of
+-1.533073	these shadows .
+-2.010194	and shiyatov 2002
+-2.010194	and shiyatov themselves
+-2.010194	and shiyatov would
+-1.533073	sample should not
+-1.533073	a similar schweingruber
+-1.533073	have similarly affected
+-1.533073	, since this
+-1.533073	limited size and
+-1.533073	i skimmed this
+-1.533073	more slowly ,
+-1.533073	way slowly get
+-1.533073	a small push
+-1.533073	because so much
+-1.533073	there's some reason
+-1.533073	from someone whose
+-1.533073	to start today
+-1.533073	cru staunchly refused
+-1.533073	not struggling against
+-1.533073	multiproxy studies that
+-1.533073	subsequent study ,
+-1.533073	this study .
+-1.533073	i stumbled upon
+-1.533073	of subfossil larches
+-1.533073	the subfossil collection
+-1.533073	yamal subfossil data
+-1.533073	every subsequent study
+-1.533073	this subset in
+-1.533073	" success .
+-1.533073	taymir supplement .
+-1.533073	were supplemented by
+-1.533073	a surface ,
+-1.533073	would take an
+-1.533073	only taken a
+-1.834103	the taymir data
+-1.834103	the taymir supplement
+-1.533073	and temperature ,
+-1.533073	ghastly tendency to
+-1.533073	in terms of
+-1.533073	trees than the
+-1.533073	" that characterizes
+-1.533073	admit that the
+-1.533073	and that way
+-1.533073	is that he
+-1.533073	made that wise
+-1.533073	method that they
+-1.533073	noticed that the
+-1.533073	people that voted
+-1.533073	point that his
+-1.533073	position that the
+-1.533073	probable that the
+-1.533073	remarked that "
+-1.533073	studies that have
+-1.533073	things that cast
+-1.533073	" the trouble
+-2.010194	, the more
+-2.010194	, the resulting
+-2.010194	, the yamal
+-1.533073	. the cru
+-1.834103	<s> the subfossil
+-1.834103	<s> the yamal
+-1.533073	affected the "
+-1.533073	along the path
+-2.010194	and the people
+-2.010194	and the phil
+-2.010194	and the right
+-1.533073	archive the data
+-1.533073	at the crossroads
+-0.8145491	between the two
+-1.533073	but the further
+-1.834103	by the addition
+-1.834103	by the magnitude
+-1.533073	combine the multi-parters
+-1.533073	control the national
+-0.8145491	get the arkive
+-1.533073	given the use
+-1.533073	has the virtue
+-1.533073	have the same
+-1.533073	if the non-robustness
+-1.834103	in the realm
+-1.834103	in the schweingruber
+-1.533073	including the taymir
+-1.533073	into the future
+-1.533073	is the most
+-1.533073	makes the point
+-0.8145491	of the 12
+-1.533073	on the trouble
+-2.010194	perhaps the biased
+-2.010194	perhaps the day
+-2.010194	perhaps the difference
+-1.533073	precisely the right
+-1.533073	provide the measurement
+-1.834103	see the far
+-1.834103	see the shadows
+-1.533073	seized the nettle
+-1.533073	than the schweingruber
+-2.135133	that the conservatives
+-2.135133	that the cru
+-2.135133	that the previous
+-2.135133	that the yamal
+-1.533073	through the very
+-2.135133	to the back-and-forth
+-2.135133	to the cru
+-2.135133	to the large-scale
+-2.135133	to the usual
+-1.533073	took the position
+-1.533073	up the biblical
+-1.533073	used the chronology
+-1.533073	using the schweingruber
+-1.533073	were the longest
+-1.533073	what the conservatives
+-1.533073	while the yamal
+-2.135133	with the information
+-2.135133	with the rcs
+-2.135133	with the taymir
+-2.135133	with the yamal
+-1.533073	wrote the following:
+-1.533073	how their cores
+-1.533073	shiyatov themselves ,
+-1.533073	they themselves were
+-1.533073	perhaps there's some
+-1.834103	. these data
+-1.834103	. these were
+-1.533073	cast these shadows
+-1.533073	notwithstanding these warnings
+-1.533073	and they can
+-1.533073	did they expect
+-1.533073	perhaps they don't
+-1.533073	that they themselves
+-1.533073	) things that
+-1.533073	several things caught
+-1.533073	to think up
+-1.533073	fully thinking through
+-2.010194	, this analysis
+-2.010194	, this chronology
+-2.010194	, this will
+-1.834103	. this bias
+-1.834103	. this is
+-1.533073	<s> this morning
+-1.533073	but this is
+-1.533073	earlier this year
+-1.533073	in this refusal
+-1.533073	of this subset
+-1.834103	on this difference
+-1.834103	on this study
+-1.533073	presented this chronology
+-1.533073	since this method
+-1.533073	skimmed this article
+-1.533073	upon this piece
+-1.533073	all those years
+-1.533073	to those "
+-1.533073	provocative thought .
+-1.533073	a thousand ,
+-1.533073	thinking through the
+-1.533073	a time ,
+-1.533073	right time and
+-1.533073	approach to constructing
+-1.533073	are to those
+-1.533073	briffa to archive
+-1.533073	but to what
+-1.533073	compared to the
+-1.533073	day to day
+-1.533073	down to about
+-1.533073	exception to the
+-1.533073	going to start
+-1.533073	hard to think
+-1.533073	hate to admit
+-1.533073	him to begin
+-1.533073	just to the
+-1.533073	love to get
+-1.533073	power to change
+-1.533073	react to what
+-1.533073	refused to provide
+-1.533073	related to different
+-1.533073	response to the
+-1.533073	tendency to mix
+-1.533073	trying to control
+-1.533073	where to begin
+-1.533073	happens today would
+-1.533073	start today .
+-1.533073	) took the
+-1.533073	bias towards older
+-1.834103	phil trans b
+-1.834103	phil trans editors
+-1.533073	old trees described
+-2.010194	older trees .
+-2.010194	older trees an
+-2.010194	older trees than
+-0.8145491	the trouble with
+-1.533073	being true ,
+-1.533073	like trying to
+-0.8145491	the two versions
+-1.533073	remained unarchived .
+-1.533073	arkive under control
+-1.533073	an unintentional bias
+-1.533073	potential unrepresentativeness of
+-1.533073	, until recently
+-1.533073	violence unveiled: humanity
+-1.533073	looking up the
+-1.533073	think up a
+-1.533073	stumbled upon this
+-1.533073	initial use of
+-1.533073	the use of
+-1.533073	" used by
+-1.533073	briffa used the
+-1.533073	data used in
+-1.533073	got used in
+-1.533073	was used in
+-1.533073	, using the
+-1.533073	not using .
+-1.533073	the usual predictable
+-1.533073	a valid reason
+-1.533073	centennial-scale variability and
+-1.533073	interannual variability .
+-1.834103	two versions .
+-1.834103	two versions is
+-1.533073	it's very hard
+-1.533073	the very limited
+-1.533073	. violence unveiled:
+-1.533073	in virtually every
+-1.533073	the virtue of
+-1.533073	that voted for
+-1.533073	methodology warn against
+-1.533073	these warnings ,
+-1.533073	blade was like
+-1.533073	book was ghostwritten
+-1.533073	chronology was used
+-1.533073	data was finally
+-1.533073	it was never
+-1.533073	that way slowly
+-1.533073	for we do
+-1.533073	may well have
+-1.533073	conservatives were right
+-1.533073	cores were selected
+-1.533073	data were supplemented
+-1.533073	larches were selected
+-1.533073	themselves were not
+-1.533073	these were the
+-1.533073	. what did
+-1.533073	<s> what a
+-1.533073	changing what happens
+-1.533073	doubt what paul
+-1.533073	exactly what the
+-1.834103	to what is
+-1.834103	to what will
+-1.533073	. what's your
+-1.834103	, when combined
+-1.834103	, when i
+-1.533073	mind when he
+-1.533073	, where sensitivity
+-1.533073	case where it's
+-1.533073	know where to
+-1.533073	( which had
+-1.834103	, which ,
+-1.834103	, which makes
+-1.533073	method which did
+-1.533073	( while looking
+-1.533073	, while including
+-1.533073	. while the
+-1.533073	someone whose book
+-1.533073	reason why schweingruber's
+-1.533073	ring widths and
+-1.533073	this will have
+-1.533073	what will be
+-1.533073	that wise crack
+-1.533073	. with the
+-1.533073	begin with .
+-1.533073	change with a
+-1.533073	chronology with its
+-1.533073	combination with the
+-1.533073	combined with the
+-1.533073	connection with osborn
+-1.533073	included with the
+-1.533073	originated with briffa
+-0.8145491	trouble with obama
+-1.533073	is within your
+-1.533073	done without fully
+-1.533073	always worth reading
+-1.533073	bias would not
+-1.533073	he would do
+-1.533073	shiyatov would not
+-1.533073	today would take
+-1.533073	. wright's church
+-1.533073	he wrote the
+-1.533073	, yamal larch
+-1.533073	briffa's yamal reconstruction
+-1.533073	resulting yamal chronology
+-1.212489	the yamal chronology
+-2.232043	the yamal data
+-2.232043	the yamal measurement
+-2.232043	the yamal subfossil
+-1.533073	200–400 year old
+-1.533073	this year ,
+-1.533073	those years ?
+-1.533073	' yes ,
+-1.533073	back-and-forth yesterday about
+-1.533073	it yet ,
+-1.533073	ahead you see
+-1.533073	what's your great
+-1.533073	within your power
+
+\end\
diff --git a/decoder/test_data/grammar.prune b/decoder/test_data/grammar.prune
new file mode 100644
index 00000000..4ebcb509
--- /dev/null
+++ b/decoder/test_data/grammar.prune
@@ -0,0 +1,196 @@
+[PHRASE] ||| [PHRASE,1] haus ||| [PHRASE,1] house ||| 1.86183 0 0 0 0.0211892
+[PHRASE] ||| [PHRASE,1] haus ist ||| is [PHRASE,1] house ||| 2.58883 0.311249 0 0.348455 0.0211893
+[PHRASE] ||| [PHRASE,1] haus gibt ||| is [PHRASE,1] house ||| 2.56863 0.291046 0 0.258278 0.0211893
+[PHRASE] ||| [PHRASE,1] ein haus ist ||| [PHRASE,1] is a house ||| 3.16286 0 0 0.576934 0.0211893
+[PHRASE] ||| [PHRASE,1] ist ||| [PHRASE,1] is ||| 2.94101 0 0.676694 0.348455 0
+[PHRASE] ||| [PHRASE,1] ist ||| is [PHRASE,1] ||| 2.36698 0.649056 0.102662 0.348455 0
+[PHRASE] ||| [PHRASE,1] klein ist ||| [PHRASE,1] is small ||| 2.58883 0.124939 0 0.78211 0
+[PHRASE] ||| [PHRASE,1] maus ||| [PHRASE,1] mouse ||| 2.09592 0 0 0 0
+[PHRASE] ||| [PHRASE,1] maus gibt ||| is [PHRASE,1] mouse ||| 2.44865 0 0 0.258278 0
+[PHRASE] ||| [PHRASE,1] kleines ||| [PHRASE,1] small ||| 2.94101 0.439333 0 0.579784 0
+[PHRASE] ||| [PHRASE,1] kleines haus ||| [PHRASE,1] small house ||| 3.24204 0 0 0.579784 0.0211893
+[PHRASE] ||| [PHRASE,1] kleines haus gibt ||| is [PHRASE,1] small house ||| 3.30899 0 0 0.838062 0.0211893
+[PHRASE] ||| [PHRASE,1] kleine ||| [PHRASE,1] small ||| 2.94101 0.439333 0 0.500602 0
+[PHRASE] ||| [PHRASE,1] kleine maus ||| [PHRASE,1] small mouse ||| 3.24204 0 0 0.500602 0
+[PHRASE] ||| [PHRASE,1] kleine maus gibt ||| is [PHRASE,1] small mouse ||| 3.30899 0 0 0.75888 0
+[PHRASE] ||| [PHRASE,1] gelb ||| [PHRASE,1] yellow ||| 2.63998 0 0 0 0
+[PHRASE] ||| [PHRASE,1] gelb haus ||| [PHRASE,1] yellow house ||| 3.24204 0 0 0 0.0211893
+[PHRASE] ||| [PHRASE,1] gelb haus gibt ||| is [PHRASE,1] yellow house ||| 3.30899 0 0 0.258278 0.0211893
+[PHRASE] ||| [PHRASE,1] gelb maus ||| [PHRASE,1] yellow mouse ||| 3.24204 0 0 0 0
+[PHRASE] ||| [PHRASE,1] gelb maus gibt ||| is [PHRASE,1] yellow mouse ||| 3.30899 0 0 0.258278 0
+[PHRASE] ||| [PHRASE,1] gibt ||| is [PHRASE,1] ||| 1.82827 0.110339 0 0.258278 0
+[PHRASE] ||| haus ||| small yellow mouse house ||| 2.46389 0.845098 1.30103 0.278754 1.34341
+[PHRASE] ||| haus ||| house ||| Phrase_0=1.18514 Phrase_2=0.0222764 Phrase_4=0.0211893
+[PHRASE] ||| haus [PHRASE,1] ||| house [PHRASE,1] ||| 2.2878 0 0 0 0.0211893
+[PHRASE] ||| haus ist ||| house is ||| 2.46389 0 0 0.348455 0.0211893
+[PHRASE] ||| haus klein ist ||| house is small ||| 2.2878 0 0 0.78211 0.0211893
+[PHRASE] ||| ein ||| a ||| Phrase_0=1.34995 Phrase_1=0.228479 Phrase_3=0.228479
+[PHRASE] ||| ein [PHRASE,1] ||| a [PHRASE,1] ||| 2.03792 0.290035 0 0.228479 0
+[PHRASE] ||| ein [PHRASE,1] haus ||| a [PHRASE,1] house ||| 2.94101 0 0 0.228479 0.0211893
+[PHRASE] ||| ein [PHRASE,1] haus gibt ||| is a [PHRASE,1] house ||| 3.00796 0 0 0.486757 0.0211893
+[PHRASE] ||| ein [PHRASE,1] ist ||| is a [PHRASE,1] ||| 2.58883 0.535113 0 0.576934 0
+[PHRASE] ||| ein [PHRASE,1] gibt ||| is a [PHRASE,1] ||| 2.56863 0.51491 0 0.486757 0
+[PHRASE] ||| ein haus ||| a house ||| 1.76492 0 0.0791813 0.228479 0.0211893
+[PHRASE] ||| ein haus ||| a small house ||| 2.46389 0.30103 0.778151 0.507233 1.34341
+[PHRASE] ||| ein haus ist ||| is a house ||| 2.76492 0.477121 0 0.576934 0.0211893
+[PHRASE] ||| ein haus gibt ||| is a house ||| 2.46389 0.176091 0.176091 0.486757 0.0211893
+[PHRASE] ||| ein haus gibt ||| is a small house ||| 2.76492 0.39794 0.477121 0.765511 1.34341
+[PHRASE] ||| ein kleines ||| a small ||| 1.86183 0.243038 0 0.808263 0
+[PHRASE] ||| ein kleines [PHRASE,1] ||| a small [PHRASE,1] ||| 3.24204 0.30103 0 0.808263 0
+[PHRASE] ||| ein kleines [PHRASE,1] gibt ||| is a small [PHRASE,1] ||| 3.30899 0.30103 0 1.06654 0
+[PHRASE] ||| ein kleines haus ||| a small house ||| 2.46389 0.30103 0 0.808263 0.0211893
+[PHRASE] ||| ein kleines haus ist ||| is a small house ||| 2.76492 0.39794 0 1.15672 0.0211893
+[PHRASE] ||| ein kleines haus gibt ||| is a small house ||| 3.06595 0.69897 0 1.06654 0.0211893
+[PHRASE] ||| ein kleines gelb ||| a small yellow ||| 2.94101 0.30103 0 0.808263 0
+[PHRASE] ||| ein kleines gelb haus ||| a small yellow house ||| 3.24204 0 0 0.808263 0.0211893
+[PHRASE] ||| ein kleines gelb haus gibt ||| is a small yellow house ||| 3.30899 0 0 1.06654 0.0211893
+[PHRASE] ||| ein gelb ||| a yellow ||| 1.98677 0.221849 0 0.228479 0
+[PHRASE] ||| ein gelb [PHRASE,1] ||| a yellow [PHRASE,1] ||| 3.24204 0.30103 0 0.228479 0
+[PHRASE] ||| ein gelb [PHRASE,1] gibt ||| is a yellow [PHRASE,1] ||| 3.30899 0.30103 0 0.486757 0
+[PHRASE] ||| ein gelb haus ||| a yellow house ||| 2.63998 0 0 0.228479 0.0211893
+[PHRASE] ||| ein gelb haus ist ||| is a yellow house ||| 3.06595 0.30103 0 0.576934 0.0211893
+[PHRASE] ||| ein gelb haus gibt ||| is a yellow house ||| 3.06595 0.30103 0 0.486757 0.0211893
+[PHRASE] ||| ein gelb kleines ||| a yellow small ||| 2.94101 0.30103 0 0.808263 0
+[PHRASE] ||| ein gelb kleines haus ||| a yellow small house ||| 3.24204 0 0 0.808263 0.0211893
+[PHRASE] ||| ein gelb kleines haus gibt ||| is a yellow small house ||| 3.30899 0 0 1.06654 0.0211893
+[PHRASE] ||| ist ||| is ||| 1.34995 0.348455 0 0.348455 0
+[PHRASE] ||| klein ||| small ||| 1.61879 0.410174 0 0.433656 0
+[PHRASE] ||| klein [PHRASE,1] ||| [PHRASE,1] small ||| 3.06595 0.564271 0 0.433656 0
+[PHRASE] ||| klein [PHRASE,1] ist ||| [PHRASE,1] is small ||| 3.06595 0.60206 0 0.78211 0
+[PHRASE] ||| klein ist ||| is small ||| 1.68574 0 0 0.78211 0
+[PHRASE] ||| klein das [PHRASE,1] ||| the [PHRASE,1] small ||| 3.06595 0 0 0.433656 0.30103
+[PHRASE] ||| klein das haus ist ||| the house is small ||| 3.06595 0.477121 0 0.78211 0.322219
+[PHRASE] ||| maus ||| mouse ||| 1.50965 0 0 0 0
+[PHRASE] ||| maus [PHRASE,1] ||| mouse [PHRASE,1] ||| 2.94101 0 0 0 0
+[PHRASE] ||| maus [PHRASE,1] ist ||| mouse is [PHRASE,1] ||| 2.94101 0 0 0.348455 0
+[PHRASE] ||| maus ein haus ist ||| mouse is a house ||| 2.94101 0 0 0.576934 0.0211893
+[PHRASE] ||| kleines ||| small ||| 1.76492 0.556302 0 0.579784 0
+[PHRASE] ||| kleines [PHRASE,1] ||| small [PHRASE,1] ||| 2.94101 0.30103 0 0.579784 0
+[PHRASE] ||| kleines haus ||| small house ||| 1.86183 0.243038 0 0.579784 0.0211893
+[PHRASE] ||| kleines gelb ||| small yellow ||| 2.46389 0.30103 0 0.579784 0
+[PHRASE] ||| kleines gelb haus ||| small yellow house ||| 2.94101 0 0 0.579784 0.0211893
+[PHRASE] ||| kleine ||| small ||| 1.68574 0.477121 0 0.500602 0
+[PHRASE] ||| kleine [PHRASE,1] ||| small [PHRASE,1] ||| 2.94101 0.30103 0 0.500602 0
+[PHRASE] ||| kleine haus ||| small house ||| 2.16286 0.544068 0 0.500602 0.0211893
+[PHRASE] ||| kleine maus ||| small mouse ||| 1.98677 0 0 0.500602 0
+[PHRASE] ||| kleine gelb ||| small yellow ||| 2.46389 0.30103 0 0.500602 0
+[PHRASE] ||| kleine gelb maus ||| small yellow mouse ||| 2.94101 0 0 0.500602 0
+[PHRASE] ||| gelb ||| yellow ||| 1.61879 0 0 0 0
+[PHRASE] ||| gelb [PHRASE,1] ||| yellow [PHRASE,1] ||| 2.63998 0 0 0 0
+[PHRASE] ||| gelb haus ||| yellow house ||| 1.98677 0 0 0 0.0211893
+[PHRASE] ||| gelb maus ||| yellow mouse ||| 2.16286 0 0 0 0
+[PHRASE] ||| gelb kleines ||| yellow small ||| 2.46389 0.30103 0 0.579784 0
+[PHRASE] ||| gelb kleines haus ||| yellow small house ||| 2.94101 0 0 0.579784 0.0211893
+[PHRASE] ||| gelb kleine ||| yellow small ||| 2.46389 0.30103 0 0.500602 0
+[PHRASE] ||| gelb kleine maus ||| yellow small mouse ||| 2.94101 0 0 0.500602 0
+[PHRASE] ||| eine ||| a ||| 1.50965 0.38818 0 0.38818 0
+[PHRASE] ||| eine [PHRASE,1] ||| a [PHRASE,1] ||| 2.0602 0.312311 0 0.38818 0
+[PHRASE] ||| eine [PHRASE,1] maus ||| a [PHRASE,1] mouse ||| 2.94101 0 0 0.38818 0
+[PHRASE] ||| eine [PHRASE,1] maus gibt ||| is a [PHRASE,1] mouse ||| 3.00796 0 0 0.646458 0
+[PHRASE] ||| eine [PHRASE,1] gibt ||| is a [PHRASE,1] ||| 2.44865 0.394934 0 0.646458 0
+[PHRASE] ||| eine maus ||| a mouse ||| 1.98677 0 0 0.38818 0
+[PHRASE] ||| eine maus [PHRASE,1] ||| a mouse [PHRASE,1] ||| 3.16286 0 0 0.38818 0
+[PHRASE] ||| eine maus [PHRASE,1] ist ||| a mouse is [PHRASE,1] ||| 3.16286 0 0 0.736635 0
+[PHRASE] ||| eine maus ein haus ist ||| a mouse is a house ||| 3.16286 0 0 0.965114 0.0211893
+[PHRASE] ||| eine maus gibt ||| is a mouse ||| 2.46389 0 0 0.646458 0
+[PHRASE] ||| eine kleine ||| a small ||| 1.98677 0.367977 0 0.888783 0
+[PHRASE] ||| eine kleine [PHRASE,1] ||| a small [PHRASE,1] ||| 3.24204 0.30103 0 0.888783 0
+[PHRASE] ||| eine kleine [PHRASE,1] gibt ||| is a small [PHRASE,1] ||| 3.30899 0.30103 0 1.14706 0
+[PHRASE] ||| eine kleine maus ||| a small mouse ||| 2.63998 0 0 0.888783 0
+[PHRASE] ||| eine kleine maus gibt ||| is a small mouse ||| 2.76492 0 0 1.14706 0
+[PHRASE] ||| eine kleine gelb ||| a small yellow ||| 2.94101 0.30103 0 0.888783 0
+[PHRASE] ||| eine kleine gelb maus ||| a small yellow mouse ||| 3.24204 0 0 0.888783 0
+[PHRASE] ||| eine kleine gelb maus gibt ||| is a small yellow mouse ||| 3.30899 0 0 1.14706 0
+[PHRASE] ||| eine gelb ||| a yellow ||| 2.16286 0.39794 0 0.38818 0
+[PHRASE] ||| eine gelb [PHRASE,1] ||| a yellow [PHRASE,1] ||| 3.24204 0.30103 0 0.38818 0
+[PHRASE] ||| eine gelb [PHRASE,1] gibt ||| is a yellow [PHRASE,1] ||| 3.30899 0.30103 0 0.646458 0
+[PHRASE] ||| eine gelb maus ||| a yellow mouse ||| 2.94101 0 0 0.38818 0
+[PHRASE] ||| eine gelb maus gibt ||| is a yellow mouse ||| 3.06595 0 0 0.646458 0
+[PHRASE] ||| eine gelb kleine ||| a yellow small ||| 2.94101 0.30103 0 0.888783 0
+[PHRASE] ||| eine gelb kleine maus ||| a yellow small mouse ||| 3.24204 0 0 0.888783 0
+[PHRASE] ||| eine gelb kleine maus gibt ||| is a yellow small mouse ||| 3.30899 0 0 1.14706 0
+[PHRASE] ||| eine gruen ||| a green ||| 2.46389 0 0 0.38818 0
+[PHRASE] ||| eine gruen maus ||| a green mouse ||| 2.94101 0 0 0.38818 0
+[PHRASE] ||| gruen ||| green ||| 2.16286 0 0 0 0
+[PHRASE] ||| gruen maus ||| green mouse ||| 2.46389 0 0 0 0
+[PHRASE] ||| tages ||| day ||| 2.46389 0 0 0 0
+[PHRASE] ||| gibt ||| is ||| 1.25977 0.258278 0 0.258278 0
+[PHRASE] ||| meins ||| mine ||| 2.16286 0 0 0 0
+[PHRASE] ||| meins [PHRASE,1] ||| mine [PHRASE,1] ||| 2.76492 0 0 0 0
+[PHRASE] ||| meins ist ||| is mine ||| 2.46389 0 0 0.348455 0
+[PHRASE] ||| meins klein ist ||| mine is small ||| 2.76492 0 0 0.78211 0
+[PHRASE] ||| geld ||| money ||| 1.98677 0 0 0 0
+[PHRASE] ||| geld ist ||| is money ||| 2.46389 0.30103 0 0.348455 0
+[PHRASE] ||| geld gibt ||| is money ||| 2.46389 0.30103 0 0.258278 0
+[PHRASE] ||| keins ||| none ||| 1.98677 0 0 0 0
+[PHRASE] ||| keins [PHRASE,1] ||| none [PHRASE,1] ||| 2.76492 0 0 0 0
+[PHRASE] ||| keins klein ist ||| none is small ||| 2.76492 0 0 0.78211 0
+[PHRASE] ||| keins gibt ||| is none ||| 2.46389 0 0 0.258278 0
+[PHRASE] ||| dem haeuschen ||| of control ||| 2.46389 0 0 0.681241 0.425969
+[PHRASE] ||| eines ||| one ||| 2.46389 0.30103 0 0.30103 0
+[PHRASE] ||| eines tages ||| one day ||| 2.46389 0 0 0.30103 0
+[PHRASE] ||| eins ||| one ||| 2.46389 0.30103 0 0.30103 0
+[PHRASE] ||| aus ||| out ||| 2.46389 0 0.477121 0 0.221849
+[PHRASE] ||| aus ||| out of ||| 2.16286 0 0.176091 0.0791812 0.619789
+[PHRASE] ||| aus [PHRASE,1] ||| out [PHRASE,1] ||| 2.76492 0 0.367977 0 0.221849
+[PHRASE] ||| aus [PHRASE,1] ||| out of [PHRASE,1] ||| 2.63998 0 0.243038 0.0791812 0.619789
+[PHRASE] ||| aus ein ||| out of a ||| 2.46389 0 0 0.307661 0.619789
+[PHRASE] ||| aus ein haus ||| out of a house ||| 2.94101 0 0 0.307661 0.640978
+[PHRASE] ||| aus dem haeuschen ||| out of control ||| 2.76492 0 0 0.681241 0.647817
+[PHRASE] ||| aus das ||| out of the ||| 2.46389 0 0 0.0791812 0.920819
+[PHRASE] ||| aus das haus ||| out of the house ||| 2.94101 0 0 0.0791812 0.942008
+[PHRASE] ||| das ||| the ||| 1.76492 0 0.30103 0 0.30103
+[PHRASE] ||| das ||| that ||| 1.76492 0 0.30103 0 0.30103
+[PHRASE] ||| das [PHRASE,1] ||| the [PHRASE,1] ||| 2.39695 0 0.41972 0 0.30103
+[PHRASE] ||| das [PHRASE,1] ||| that [PHRASE,1] ||| 2.18514 0 0.207913 0 0.30103
+[PHRASE] ||| das [PHRASE,1] haus ist ||| that is [PHRASE,1] house ||| 2.86183 0 0 0.348455 0.322219
+[PHRASE] ||| das [PHRASE,1] ist ||| that is [PHRASE,1] ||| 2.86183 0 0 0.348455 0.30103
+[PHRASE] ||| das haus ||| the house ||| 1.86183 0 0 0 0.322219
+[PHRASE] ||| das haus [PHRASE,1] ||| the house [PHRASE,1] ||| 2.76492 0 0 0 0.322219
+[PHRASE] ||| das haus ist ||| the house is ||| 2.94101 0 0 0.348455 0.322219
+[PHRASE] ||| das haus klein ist ||| the house is small ||| 2.76492 0.176091 0 0.78211 0.322219
+[PHRASE] ||| das ein [PHRASE,1] ist ||| that is a [PHRASE,1] ||| 2.86183 0 0 0.576934 0.30103
+[PHRASE] ||| das ein kleines haus ist ||| that is a small house ||| 3.16286 0 0 1.15672 0.322219
+[PHRASE] ||| das ein gelb haus ist ||| that is a yellow house ||| 3.16286 0 0 0.576934 0.322219
+[PHRASE] ||| das klein ist ||| that is small ||| 2.76492 0 0 0.78211 0.30103
+[PHRASE] ||| das kleine ||| the small ||| 2.46389 0 0 0.500602 0.30103
+[PHRASE] ||| das kleine haus ||| the small house ||| 2.94101 0 0 0.500602 0.322219
+[PHRASE] ||| das meins ist ||| that is mine ||| 2.76492 0 0 0.348455 0.30103
+[PHRASE] ||| das geld ist ||| that is money ||| 2.76492 0 0 0.348455 0.30103
+[PHRASE] ||| es ||| there ||| 1.25977 0 0 0 0
+[PHRASE] ||| es [PHRASE,1] ||| there [PHRASE,1] ||| 1.83672 0 0 0 0
+[PHRASE] ||| es [PHRASE,1] haus gibt ||| there is [PHRASE,1] house ||| 2.62775 0 0 0.258278 0.0211893
+[PHRASE] ||| es [PHRASE,1] maus gibt ||| there is [PHRASE,1] mouse ||| 2.5166 0 0 0.258278 0
+[PHRASE] ||| es [PHRASE,1] kleines haus gibt ||| there is [PHRASE,1] small house ||| 3.30899 0 0 0.838062 0.0211893
+[PHRASE] ||| es [PHRASE,1] kleine maus gibt ||| there is [PHRASE,1] small mouse ||| 3.30899 0 0 0.75888 0
+[PHRASE] ||| es [PHRASE,1] gelb haus gibt ||| there is [PHRASE,1] yellow house ||| 3.30899 0 0 0.258278 0.0211893
+[PHRASE] ||| es [PHRASE,1] gelb maus gibt ||| there is [PHRASE,1] yellow mouse ||| 3.30899 0 0 0.258278 0
+[PHRASE] ||| es [PHRASE,1] gibt ||| there is [PHRASE,1] ||| 1.9536 0 0 0.258278 0
+[PHRASE] ||| es ein [PHRASE,1] haus gibt ||| there is a [PHRASE,1] house ||| 3.00796 0 0 0.486757 0.0211893
+[PHRASE] ||| es ein [PHRASE,1] gibt ||| there is a [PHRASE,1] ||| 2.62775 0.360151 0 0.486757 0
+[PHRASE] ||| es ein haus gibt ||| there is a house ||| 2.63998 0 0.176091 0.486757 0.0211893
+[PHRASE] ||| es ein haus gibt ||| there is a small house ||| 2.94101 0.20412 0.477121 0.765511 1.34341
+[PHRASE] ||| es ein kleines [PHRASE,1] gibt ||| there is a small [PHRASE,1] ||| 3.30899 0.30103 0 1.06654 0
+[PHRASE] ||| es ein kleines haus gibt ||| there is a small house ||| 3.16286 0.425969 0 1.06654 0.0211893
+[PHRASE] ||| es ein gelb [PHRASE,1] gibt ||| there is a yellow [PHRASE,1] ||| 3.30899 0.30103 0 0.486757 0
+[PHRASE] ||| es ein gelb haus gibt ||| there is a yellow house ||| 3.16286 0 0 0.486757 0.0211893
+[PHRASE] ||| es eine [PHRASE,1] maus gibt ||| there is a [PHRASE,1] mouse ||| 3.00796 0 0 0.646458 0
+[PHRASE] ||| es eine [PHRASE,1] gibt ||| there is a [PHRASE,1] ||| 2.5166 0.249001 0 0.646458 0
+[PHRASE] ||| es eine maus gibt ||| there is a mouse ||| 2.63998 0 0 0.646458 0
+[PHRASE] ||| es eine kleine [PHRASE,1] gibt ||| there is a small [PHRASE,1] ||| 3.30899 0.30103 0 1.14706 0
+[PHRASE] ||| es eine kleine maus gibt ||| there is a small mouse ||| 2.86183 0 0 1.14706 0
+[PHRASE] ||| es eine gelb [PHRASE,1] gibt ||| there is a yellow [PHRASE,1] ||| 3.30899 0.30103 0 0.646458 0
+[PHRASE] ||| es eine gelb maus gibt ||| there is a yellow mouse ||| 3.16286 0 0 0.646458 0
+[PHRASE] ||| es geld gibt ||| there is money ||| 2.76492 0 0 0.258278 0
+[PHRASE] ||| es keins gibt ||| there is none ||| 2.76492 0 0 0.258278 0
+[PHRASE] ||| dieses ||| this ||| 1.98677 0 0 0 0
+[PHRASE] ||| dieses [PHRASE,1] ||| this [PHRASE,1] ||| 2.56995 0 0 0 0
+[PHRASE] ||| dieses [PHRASE,1] haus ist ||| this is [PHRASE,1] house ||| 3.16286 0 0 0.348455 0.0211893
+[PHRASE] ||| dieses [PHRASE,1] ist ||| this is [PHRASE,1] ||| 3.16286 0 0 0.348455 0
+[PHRASE] ||| dieses haus ||| this house ||| 2.46389 0 0 0 0.0211893
+[PHRASE] ||| dieses haus [PHRASE,1] ||| this house [PHRASE,1] ||| 3.06595 0 0 0 0.0211893
+[PHRASE] ||| dieses haus klein ist ||| this house is small ||| 3.06595 0 0 0.78211 0.0211893
+[PHRASE] ||| dieses ein [PHRASE,1] ist ||| this is a [PHRASE,1] ||| 3.16286 0 0 0.576934 0
+[PHRASE] ||| dieses ein kleines haus ist ||| this is a small house ||| 3.16286 0 0 1.15672 0.0211893
+[PHRASE] ||| dieses kleine ||| this small ||| 2.46389 0 0 0.500602 0
+[PHRASE] ||| dieses kleine haus ||| this small house ||| 2.94101 0 0 0.500602 0.0211893
diff --git a/decoder/test_data/small.json.gz b/decoder/test_data/small.json.gz
new file mode 100644
index 00000000..892ba360
--- /dev/null
+++ b/decoder/test_data/small.json.gz
diff --git a/decoder/test_data/test_2gram.lm.gz b/decoder/test_data/test_2gram.lm.gz
new file mode 100644
index 00000000..aafa7274
--- /dev/null
+++ b/decoder/test_data/test_2gram.lm.gz
diff --git a/decoder/test_data/weights b/decoder/test_data/weights
new file mode 100644
index 00000000..ea70229c
--- /dev/null
+++ b/decoder/test_data/weights
@@ -0,0 +1,8 @@
+# hiero
+WordPenalty -0.387029
+LanguageModel 0.253195
+PhraseModel_0 0.142926
+PhraseModel_1 0.465119
+PhraseModel_2 0.079503
+CNPosteriorProbability 0.09259
+Inf -inf
diff --git a/decoder/test_data/weights.gt b/decoder/test_data/weights.gt
new file mode 100644
index 00000000..08931049
--- /dev/null
+++ b/decoder/test_data/weights.gt
@@ -0,0 +1,4 @@
+Phrase_0 1.0
+Phrase_1 0.5
+Phrase_2 0.3
+Phrase_3 0.2
diff --git a/decoder/timing_stats.cc b/decoder/timing_stats.cc
new file mode 100644
index 00000000..85b95de5
--- /dev/null
+++ b/decoder/timing_stats.cc
@@ -0,0 +1,24 @@
+#include "timing_stats.h"
+
+#include <iostream>
+
+using namespace std;
+
+map<string, TimerInfo> Timer::stats;
+
+Timer::Timer(const string& timername) : start_t(clock()), cur(stats[timername]) {}
+
+Timer::~Timer() {
+  ++cur.calls;
+  const clock_t end_t = clock();
+  const double elapsed = (end_t - start_t) / 1000000.0;
+  cur.total_time += elapsed;
+}
+
+void Timer::Summarize() {
+  for (map<string, TimerInfo>::iterator it = stats.begin(); it != stats.end(); ++it) {
+    cerr << it->first << ": " << it->second.total_time << " secs (" << it->second.calls << " calls)\n";
+  }
+  stats.clear();
+}
+
diff --git a/decoder/timing_stats.h b/decoder/timing_stats.h
new file mode 100644
index 00000000..0a9f7656
--- /dev/null
+++ b/decoder/timing_stats.h
@@ -0,0 +1,25 @@
+#ifndef _TIMING_STATS_H_
+#define _TIMING_STATS_H_
+
+#include <string>
+#include <map>
+
+struct TimerInfo {
+  int calls;
+  double total_time;
+  TimerInfo() : calls(), total_time() {}
+};
+
+struct Timer {
+  Timer(const std::string& info);
+  ~Timer();
+  static void Summarize();
+ private:
+  static std::map<std::string, TimerInfo> stats;
+  clock_t start_t;
+  TimerInfo& cur;
+  Timer(const Timer& other);
+  const Timer& operator=(const Timer& other);
+};
+
+#endif
diff --git a/decoder/translator.cc b/decoder/translator.cc
new file mode 100644
index 00000000..e6c282e1
--- /dev/null
+++ b/decoder/translator.cc
@@ -0,0 +1,57 @@
+#include "translator.h"
+
+#include <iostream>
+#include <vector>
+
+using namespace std;
+
+Translator::~Translator() {}
+
+void Translator::ProcessMarkupHints(const map<string, string>& kv) {
+  if (state_ != kUninitialized) {
+    cerr << "Translator::ProcessMarkupHints in wrong state: " << state_ << endl;
+    abort();
+  }
+  ProcessMarkupHintsImpl(kv);
+  state_ = kReadyToTranslate;
+}
+
+bool Translator::Translate(const std::string& src,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* minus_lm_forest) {
+  if (state_ == kUninitialized) {
+    cerr << "Translator::Translate(...) must not be in uninitialized state!\n";
+    abort();
+  }
+  const bool result = TranslateImpl(src, smeta, weights, minus_lm_forest);
+  state_ = kTranslated;
+  return result;
+}
+
+void Translator::SentenceComplete() {
+  if (state_ != kTranslated) {
+    cerr << "Translator::Complete in unexpected state: " << state_ << endl;
+    // not fatal
+  }
+  SentenceCompleteImpl();
+  state_ = kUninitialized;  // return to start state
+}
+
+// this may be overridden by translators that want to accept
+// metadata
+void Translator::ProcessMarkupHintsImpl(const map<string, string>& kv) {
+  int unprocessed = kv.size() - kv.count("id");
+	cerr << "Inside translator process hints\n";
+  if (unprocessed > 0) {
+    cerr << "Sentence markup contains unprocessed data:\n";
+    for (map<string, string>::const_iterator it = kv.begin(); it != kv.end(); ++it) {
+      if (it->first == "id") continue;
+      cerr << "  KEY[" << it->first << "] --> " << it->second << endl;
+    }
+    abort();
+  }
+}
+
+void Translator::SentenceCompleteImpl() {}
+
diff --git a/decoder/translator.h b/decoder/translator.h
new file mode 100644
index 00000000..6b0a02e4
--- /dev/null
+++ b/decoder/translator.h
@@ -0,0 +1,82 @@
+#ifndef _TRANSLATOR_H_
+#define _TRANSLATOR_H_
+
+#include <string>
+#include <vector>
+#include <map>
+#include <boost/shared_ptr.hpp>
+#include <boost/program_options/variables_map.hpp>
+
+class Hypergraph;
+class SentenceMetadata;
+
+// Workflow: for each sentence to be translated
+//   1) call ProcessMarkupHints(markup)
+//   2) call Translate(...)
+//   3) call SentenceComplete()
+class Translator {
+ public:
+  Translator() : state_(kUninitialized) {}
+  virtual ~Translator();
+  // returns true if goal reached, false otherwise
+  // minus_lm_forest will contain the unpruned forest. the
+  // feature values from the phrase table / grammar / etc
+  // should be in the forest already - the "late" features
+  // should not just copy values that are available without
+  // any context or computation.
+  // SentenceMetadata contains information about the sentence,
+  // but it is an input/output parameter since the Translator
+  // is also responsible for setting the value of src_len.
+  bool Translate(const std::string& src,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* minus_lm_forest);
+
+  // This is called before Translate(...) with the sentence-
+  // level markup passed in. This can be used to set sentence-
+  // specific behavior of the translator.
+  void ProcessMarkupHints(const std::map<std::string, std::string>& kv);
+
+  // Free any sentence-specific resources
+  void SentenceComplete();
+ protected:
+  virtual bool TranslateImpl(const std::string& src,
+                             SentenceMetadata* smeta,
+                             const std::vector<double>& weights,
+                             Hypergraph* minus_lm_forest) = 0;
+  virtual void ProcessMarkupHintsImpl(const std::map<std::string, std::string>& kv);
+  virtual void SentenceCompleteImpl();
+ private:
+  enum State { kUninitialized, kReadyToTranslate, kTranslated };
+  State state_;
+};
+
+class SCFGTranslatorImpl;
+class SCFGTranslator : public Translator {
+ public:
+  SCFGTranslator(const boost::program_options::variables_map& conf);
+ protected:
+  bool TranslateImpl(const std::string& src,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* minus_lm_forest);
+  void ProcessMarkupHintsImpl(const std::map<std::string, std::string>& kv);
+  void SentenceCompleteImpl();
+ private:
+  boost::shared_ptr<SCFGTranslatorImpl> pimpl_;
+};
+
+class FSTTranslatorImpl;
+class FSTTranslator : public Translator {
+ public:
+  FSTTranslator(const boost::program_options::variables_map& conf);
+ private:
+  bool TranslateImpl(const std::string& src,
+                 SentenceMetadata* smeta,
+                 const std::vector<double>& weights,
+                 Hypergraph* minus_lm_forest);
+ private:
+  boost::shared_ptr<FSTTranslatorImpl> pimpl_;
+};
+
+#endif
diff --git a/decoder/tromble_loss.cc b/decoder/tromble_loss.cc
new file mode 100644
index 00000000..9ebd8ab1
--- /dev/null
+++ b/decoder/tromble_loss.cc
@@ -0,0 +1,309 @@
+#include "tromble_loss.h"
+
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/circular_buffer.hpp>
+#include <boost/functional/hash.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/range/iterator_range.hpp>
+#include <boost/tokenizer.hpp>
+#include <boost/unordered_map.hpp>
+
+#include <cmath>
+#include <fstream>
+#include <vector>
+
+#include "sentence_metadata.h"
+#include "trule.h"
+#include "tdict.h"
+
+using namespace std;
+
+namespace {
+
+typedef unsigned char GramCount;
+
+struct RefCounts {
+  GramCount max;
+  std::vector<GramCount> refs;
+  size_t length;
+};
+
+typedef boost::unordered_map<std::vector<WordID>, size_t, boost::hash<std::vector<WordID> > > NGramMap;
+
+// Take all the n-grams in the references and stuff them into ngrams.
+void MakeNGramMapFromReferences(const vector<vector<WordID> > &references,
+                                int n,
+                                vector<RefCounts> *counts,
+                                NGramMap *ngrams) {
+  ngrams->clear();
+  std::pair<vector<WordID>, size_t> insert_me;
+  vector<WordID> &ngram = insert_me.first;
+  ngram.reserve(n);
+  size_t &id = insert_me.second;
+  id = 0;
+  for (int refi = 0; refi < references.size(); ++refi) {
+    const vector<WordID>& ref = references[refi];
+    const int s = ref.size();
+    for (int j=0; j<s; ++j) {
+      const int remaining = s-j;
+      const int k = (n < remaining ? n : remaining);
+      ngram.clear();
+      for (unsigned int i = 0; i < k; ++i) {
+        ngram.push_back(ref[j + i]);
+        std::pair<NGramMap::iterator, bool> ret(ngrams->insert(insert_me));
+        if (ret.second) {
+          counts->resize(id + 1);
+          RefCounts &ref_counts = counts->back();
+          ref_counts.max = 1;
+          ref_counts.refs.resize(references.size());
+          ref_counts.refs[refi] = 1;
+          ref_counts.length = ngram.size();
+          ++id;
+        } else {
+          RefCounts &ref_counts = (*counts)[ret.first->second];
+          ref_counts.max = std::max(ref_counts.max, ++ref_counts.refs[refi]);
+        }
+      }
+    }
+  }
+}
+
+struct MutableState {
+  MutableState(void *from, size_t n) : length(reinterpret_cast<size_t*>(from)), left(reinterpret_cast<WordID *>(length + 1)), right(left + n - 1), counts(reinterpret_cast<GramCount *>(right + n - 1)) {}
+  size_t *length;
+  WordID *left, *right;
+  GramCount *counts;
+  static size_t Size(size_t n, size_t bound_ngram_id) { return sizeof(size_t) + (n - 1) * 2 * sizeof(WordID) + bound_ngram_id * sizeof(GramCount); }
+};
+
+struct ConstState {
+  ConstState(const void *from, size_t n) : length(reinterpret_cast<const size_t*>(from)), left(reinterpret_cast<const WordID *>(length + 1)), right(left + n - 1), counts(reinterpret_cast<const GramCount *>(right + n - 1)) {}
+  const size_t *length;
+  const WordID *left, *right;
+  const GramCount *counts;
+  static size_t Size(size_t n, size_t bound_ngram_id) { return sizeof(size_t) + (n - 1) * 2 * sizeof(WordID) + bound_ngram_id * sizeof(GramCount); }
+};
+
+template <class T> struct CompatibleHashRange : public std::unary_function<const boost::iterator_range<T> &, size_t> {
+  size_t operator()(const boost::iterator_range<T> &range) const {
+    return boost::hash_range(range.begin(), range.end());
+  }
+};
+
+template <class T> struct CompatibleEqualsRange : public std::binary_function<const boost::iterator_range<T> &, const std::vector<WordID> &, size_t> {
+  size_t operator()(const boost::iterator_range<T> &range, const std::vector<WordID> &vec) const {
+    return boost::algorithm::equals(range, vec);
+  }
+  size_t operator()(const std::vector<WordID> &vec, const boost::iterator_range<T> &range) const {
+    return boost::algorithm::equals(range, vec);
+  }
+};
+
+void AddWord(const boost::circular_buffer<WordID> &segment, size_t min_length, const NGramMap &ref_grams, GramCount *counters) {
+  typedef boost::circular_buffer<WordID>::const_iterator BufferIt;
+  typedef boost::iterator_range<BufferIt> SegmentRange;
+  if (segment.size() < min_length) return;
+#if 0
+  CompatibleHashRange<BufferIt> hasher;
+  CompatibleEqualsRange<BufferIt> equals;
+  for (BufferIt seg_start(segment.end() - min_length); ; --seg_start) {
+    NGramMap::const_iterator found = ref_grams.find(SegmentRange(seg_start, segment.end()));
+    if (found == ref_grams.end()) break;
+    ++counters[found->second];
+    if (seg_start == segment.begin()) break;
+  }
+#endif
+}
+
+} // namespace
+
+class TrombleLossComputerImpl {
+ public:
+  explicit TrombleLossComputerImpl(const std::string &params) : star_(TD::Convert("<{STAR}>")) {
+    typedef boost::tokenizer<boost::char_separator<char> > Tokenizer;
+    // Argument parsing
+    std::string ref_file_name;
+    Tokenizer tok(params, boost::char_separator<char>(" "));
+    Tokenizer::iterator i = tok.begin();
+    if (i == tok.end()) {
+      std::cerr << "TrombleLossComputer needs a reference file name." << std::endl;
+      exit(1);
+    }
+    ref_file_name = *i++;
+    if (i == tok.end()) {
+      std::cerr << "TrombleLossComputer needs to know how many references." << std::endl;
+      exit(1);
+    }
+    num_refs_ = boost::lexical_cast<unsigned int>(*i++);
+    for (; i != tok.end(); ++i) {
+     thetas_.push_back(boost::lexical_cast<double>(*i));
+    }
+    if (thetas_.empty()) {
+      std::cerr << "TrombleLossComputer is pointless with no weight on n-grams." << std::endl;
+      exit(1);
+    }
+
+    // Read references file.
+    std::ifstream ref_file(ref_file_name.c_str());
+    if (!ref_file) {
+      std::cerr << "Could not open TrombleLossComputer file " << ref_file_name << std::endl;
+      exit(1);
+    }
+    std::string ref;
+    vector<vector<WordID> > references(num_refs_);
+    bound_ngram_id_ = 0;
+    for (unsigned int sentence = 0; ref_file; ++sentence) {
+      for (unsigned int refidx = 0; refidx < num_refs_; ++refidx) {
+        if (!getline(ref_file, ref)) {
+          if (refidx == 0) break;
+          std::cerr << "Short read of " << refidx << " references for sentence " << sentence << std::endl;
+          exit(1);
+        }
+        TD::ConvertSentence(ref, &references[refidx]);
+      }
+      ref_ids_.resize(sentence + 1);
+      ref_counts_.resize(sentence + 1);
+      MakeNGramMapFromReferences(references, thetas_.size(), &ref_counts_.back(), &ref_ids_.back());
+      bound_ngram_id_ = std::max(bound_ngram_id_, ref_ids_.back().size());
+    }
+  }
+
+  size_t StateSize() const {
+    // n-1 boundary words plus counts for n-grams currently rendered as bytes even though most would fit in bits.
+    // Also, this is cached by higher up classes so no need to cache here.  
+    return MutableState::Size(thetas_.size(), bound_ngram_id_);
+  }
+
+  double Traversal(
+      const SentenceMetadata &smeta,
+      const TRule &rule,
+      const vector<const void*> &ant_contexts,
+      void *out_context) const {
+    // TODO: get refs from sentence metadata.  
+    // This will require resizable features.  
+    if (smeta.GetSentenceID() >= ref_ids_.size()) {
+      std::cerr << "Sentence ID " << smeta.GetSentenceID() << " doesn't have references; there are only " << ref_ids_.size() << " references." << std::endl;
+      exit(1);
+    }
+    const NGramMap &ngrams = ref_ids_[smeta.GetSentenceID()];
+    MutableState out_state(out_context, thetas_.size());
+    memset(out_state.counts, 0, bound_ngram_id_ * sizeof(GramCount));
+    boost::circular_buffer<WordID> history(thetas_.size());
+    std::vector<const void*>::const_iterator ant_context = ant_contexts.begin();
+    *out_state.length = 0;
+    size_t pushed = 0;
+    const size_t keep = thetas_.size() - 1;
+    for (vector<WordID>::const_iterator rhs = rule.e().begin(); rhs != rule.e().end(); ++rhs) {
+      if (*rhs < 1) {
+        assert(ant_context != ant_contexts.end());
+        // Constituent
+        ConstState rhs_state(*ant_context, thetas_.size());
+        *out_state.length += *rhs_state.length;
+        {
+          GramCount *accum = out_state.counts;
+          for (const GramCount *c = rhs_state.counts; c != rhs_state.counts + ngrams.size(); ++c, ++accum) {
+            *accum += *c;
+          }
+        }
+        const WordID *w = rhs_state.left;
+        bool long_constit = true;
+        for (size_t i = 1; i <= keep; ++i, ++w) {
+          if (*w == star_) {
+            long_constit = false;
+            break;
+          }
+          history.push_back(*w);
+          if (++pushed == keep) {
+            std::copy(history.begin(), history.end(), out_state.left);
+          }
+          // Now i is the length of the history coming from this constituent.  So it needs at least i+1 words to have a cross-child add.  
+          AddWord(history, i + 1, ngrams, out_state.counts);
+        }
+        // If the consituent is shorter than thetas_.size(), then the
+        // constituent's left is the entire constituent, so history is already
+        // correct.  Otherwise, the entire right hand side is the entire
+        // history.
+        if (long_constit) {
+          history.assign(thetas_.size(), rhs_state.right, rhs_state.right + keep);
+        }
+        ++ant_context;
+      } else {
+        // Word
+        ++*out_state.length;
+        history.push_back(*rhs);
+        if (++pushed == keep) {
+          std::copy(history.begin(), history.end(), out_state.left);
+        }
+        AddWord(history, 1, ngrams, out_state.counts);
+      }
+    }
+    // Fill in left and right constituents.
+    if (pushed < keep) {
+      std::copy(history.begin(), history.end(), out_state.left);
+      for (WordID *i = out_state.left + pushed; i != out_state.left + keep; ++i) {
+        *i = star_;
+      }
+      std::copy(out_state.left, out_state.left + keep, out_state.right);
+    } else if(pushed == keep) {
+      std::copy(history.begin(), history.end(), out_state.right);
+    } else if ((pushed > keep) && !history.empty()) {
+      std::copy(history.begin() + 1, history.end(), out_state.right);
+    }
+    std::vector<RefCounts>::const_iterator ref_info = ref_counts_[smeta.GetSentenceID()].begin();
+    // Clip the counts and count matches.
+    // Indexed by reference then by length.
+    std::vector<std::vector<unsigned int> > matches(num_refs_, std::vector<unsigned int>(thetas_.size()));
+    for (GramCount *c = out_state.counts; c != out_state.counts + ngrams.size(); ++c, ++ref_info) {
+      *c = std::min(*c, ref_info->max);
+      if (*c) {
+        for (unsigned int refidx = 0; refidx < num_refs_; ++refidx) {
+          assert(ref_info->length >= 1);
+          assert(ref_info->length - 1 < thetas_.size());
+          matches[refidx][ref_info->length - 1] += std::min(*c, ref_info->refs[refidx]);
+        }
+      }
+    }
+    double best_score = 0.0;
+    for (unsigned int refidx = 0; refidx < num_refs_; ++refidx) {
+      double score = 0.0;
+      for (unsigned int j = 0; j < std::min(*out_state.length, thetas_.size()); ++j) {
+        score += thetas_[j] * static_cast<double>(matches[refidx][j]) / static_cast<double>(*out_state.length - j);
+      }
+      best_score = std::max(best_score, score);
+    }
+    return best_score;
+  }
+
+ private:
+  unsigned int num_refs_;
+  // Indexed by sentence id.
+  std::vector<NGramMap> ref_ids_;
+  // Then by id from ref_ids_.
+  std::vector<std::vector<RefCounts> > ref_counts_;
+
+  // thetas_[0] is the weight for 1-grams
+  std::vector<double> thetas_;
+
+  // All ngram ids in ref_ids_ are < this value.
+  size_t bound_ngram_id_;
+
+  const WordID star_;
+};
+
+TrombleLossComputer::TrombleLossComputer(const std::string &params) :
+    boost::base_from_member<PImpl>(new TrombleLossComputerImpl(params)),
+    FeatureFunction(boost::base_from_member<PImpl>::member->StateSize()),
+    fid_(FD::Convert("TrombleLossComputer")) {}
+
+TrombleLossComputer::~TrombleLossComputer() {}
+
+void TrombleLossComputer::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const {
+  (void) estimated_features;
+  const double loss = boost::base_from_member<PImpl>::member->Traversal(smeta, *edge.rule_, ant_contexts, out_context);
+  features->set_value(fid_, loss);
+}
diff --git a/decoder/tromble_loss.h b/decoder/tromble_loss.h
new file mode 100644
index 00000000..599a2d54
--- /dev/null
+++ b/decoder/tromble_loss.h
@@ -0,0 +1,40 @@
+#ifndef _TROMBLE_LOSS_H_
+#define _TROMBLE_LOSS_H_
+
+#include <vector>
+#include <boost/scoped_ptr.hpp>
+#include <boost/utility/base_from_member.hpp>
+
+#include "ff.h"
+#include "wordid.h"
+
+// this may not be the most elegant way to implement this computation, but since we
+// may need cube pruning and state splitting, we reuse the feature detector framework.
+// the loss is then stored in a feature #0 (which is guaranteed to have weight 0 and
+// never be a "real" feature).
+class TrombleLossComputerImpl;
+class TrombleLossComputer : private boost::base_from_member<boost::scoped_ptr<TrombleLossComputerImpl> >, public FeatureFunction {
+ private:
+  typedef boost::scoped_ptr<TrombleLossComputerImpl> PImpl;
+  typedef FeatureFunction Base;
+
+ public:
+  // String parameters are ref.txt num_ref weight1 weight2 ... weightn
+  // where ref.txt contains references on per line, with num_ref references per sentence
+  // The weights are the weight on each length n-gram.
+  explicit TrombleLossComputer(const std::string &params);
+
+  ~TrombleLossComputer();
+
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* out_context) const;
+ private:
+  const int fid_;
+};
+
+#endif
diff --git a/decoder/trule.cc b/decoder/trule.cc
new file mode 100644
index 00000000..505839c7
--- /dev/null
+++ b/decoder/trule.cc
@@ -0,0 +1,242 @@
+#include "trule.h"
+
+#include <sstream>
+
+#include "stringlib.h"
+#include "tdict.h"
+
+using namespace std;
+
+static WordID ConvertTrgString(const string& w) {
+  int len = w.size();
+  WordID id = 0;
+  // [X,0] or [0]
+  // for target rules, we ignore the category, just keep the index
+  if (len > 2 && w[0]=='[' && w[len-1]==']' && w[len-2] > '0' && w[len-2] <= '9' &&
+      (len == 3 || (len > 4 && w[len-3] == ','))) {
+    id = w[len-2] - '0';
+    id = 1 - id;
+  } else {
+    id = TD::Convert(w);
+  }
+  return id;
+}
+
+static WordID ConvertSrcString(const string& w, bool mono = false) {
+  int len = w.size();
+  // [X,0]
+  // for source rules, we keep the category and ignore the index (source rules are
+  // always numbered 1, 2, 3...
+  if (mono) {
+    if (len > 2 && w[0]=='[' && w[len-1]==']') {
+      if (len > 4 && w[len-3] == ',') {
+        cerr << "[ERROR] Monolingual rules mut not have non-terminal indices:\n  "
+             << w << endl;
+        exit(1);
+      }
+      // TODO check that source indices go 1,2,3,etc.
+      return TD::Convert(w.substr(1, len-2)) * -1;
+    } else {
+      return TD::Convert(w);
+    }
+  } else {
+    if (len > 4 && w[0]=='[' && w[len-1]==']' && w[len-3] == ',' && w[len-2] > '0' && w[len-2] <= '9') {
+      return TD::Convert(w.substr(1, len-4)) * -1;
+    } else {
+      return TD::Convert(w);
+    }
+  }
+}
+
+static WordID ConvertLHS(const string& w) {
+  if (w[0] == '[') {
+    int len = w.size();
+    if (len < 3) { cerr << "Format error: " << w << endl; exit(1); }
+    return TD::Convert(w.substr(1, len-2)) * -1;
+  } else {
+    return TD::Convert(w) * -1;
+  }
+}
+
+TRule* TRule::CreateRuleSynchronous(const std::string& rule) {
+  TRule* res = new TRule;
+  if (res->ReadFromString(rule, true, false)) return res;
+  cerr << "[ERROR] Failed to creating rule from: " << rule << endl;
+  delete res;
+  return NULL;
+}
+
+TRule* TRule::CreateRulePhrasetable(const string& rule) {
+  // TODO make this faster
+  // TODO add configuration for default NT type
+  if (rule[0] == '[') {
+    cerr << "Phrasetable rules shouldn't have a LHS / non-terminals:\n  " << rule << endl;
+    return NULL;
+  }
+  TRule* res = new TRule("[X] ||| " + rule, true, false);
+  if (res->Arity() != 0) {
+    cerr << "Phrasetable rules should have arity 0:\n  " << rule << endl;
+    delete res;
+    return NULL;
+  }
+  return res;
+}
+
+TRule* TRule::CreateRuleMonolingual(const string& rule) {
+  return new TRule(rule, false, true);
+}
+
+bool TRule::ReadFromString(const string& line, bool strict, bool mono) {
+  e_.clear();
+  f_.clear();
+  scores_.clear();
+
+  string w;
+  istringstream is(line);
+  int format = CountSubstrings(line, "|||");
+  if (strict && format < 2) {
+    cerr << "Bad rule format in strict mode:\n" << line << endl;
+    return false;
+  }
+  if (format >= 2 || (mono && format == 1)) {
+    while(is>>w && w!="|||") { lhs_ = ConvertLHS(w); }
+    while(is>>w && w!="|||") { f_.push_back(ConvertSrcString(w, mono)); }
+    if (!mono) {
+      while(is>>w && w!="|||") { e_.push_back(ConvertTrgString(w)); }
+    }
+    int fv = 0;
+    if (is) {
+      string ss;
+      getline(is, ss);
+      //cerr << "L: " << ss << endl;
+      int start = 0;
+      const int len = ss.size();
+      while (start < len) {
+        while(start < len && (ss[start] == ' ' || ss[start] == ';'))
+          ++start;
+        if (start == len) break;
+        int end = start + 1;
+        while(end < len && (ss[end] != '=' && ss[end] != ' ' && ss[end] != ';'))
+          ++end;
+        if (end == len || ss[end] == ' ' || ss[end] == ';') {
+          //cerr << "PROC: '" << ss.substr(start, end - start) << "'\n";
+          // non-named features
+          if (end != len) { ss[end] = 0; }
+          string fname = "PhraseModel_X";
+          if (fv > 9) { cerr << "Too many phrasetable scores - used named format\n"; abort(); }
+          fname[12]='0' + fv;
+          ++fv;
+          // if the feature set is frozen, this may return zero, indicating an
+          // undefined feature
+          const int fid = FD::Convert(fname);
+          if (fid)
+            scores_.set_value(fid, atof(&ss[start]));
+          //cerr << "F: " << fname << " VAL=" << scores_.value(FD::Convert(fname)) << endl;
+        } else {
+          const int fid = FD::Convert(ss.substr(start, end - start));
+          start = end + 1;
+          end = start + 1;
+          while(end < len && (ss[end] != ' ' && ss[end] != ';'))
+            ++end;
+          if (end < len) { ss[end] = 0; }
+	  assert(start < len);
+          if (fid)
+            scores_.set_value(fid, atof(&ss[start]));
+          //cerr << "F: " << FD::Convert(fid) << " VAL=" << scores_.value(fid) << endl;
+        }
+        start = end + 1;
+      }
+    }
+  } else if (format == 1) {
+    while(is>>w && w!="|||") { lhs_ = ConvertLHS(w); }
+    while(is>>w && w!="|||") { e_.push_back(ConvertTrgString(w)); }
+    f_ = e_;
+    int x = ConvertLHS("[X]");
+    for (int i = 0; i < f_.size(); ++i)
+      if (f_[i] <= 0) { f_[i] = x; }
+  } else {
+    cerr << "F: " << format << endl;
+    cerr << "[ERROR] Don't know how to read:\n" << line << endl;
+  }
+  if (mono) {
+    e_ = f_;
+    int ci = 0;
+    for (int i = 0; i < e_.size(); ++i)
+      if (e_[i] < 0)
+        e_[i] = ci--;
+  }
+  ComputeArity();
+  return SanityCheck();
+}
+
+bool TRule::SanityCheck() const {
+  vector<int> used(f_.size(), 0);
+  int ac = 0;
+  for (int i = 0; i < e_.size(); ++i) {
+    int ind = e_[i];
+    if (ind > 0) continue;
+    ind = -ind;
+    if ((++used[ind]) != 1) {
+      cerr << "[ERROR] e-side variable index " << (ind+1) << " used more than once!\n";
+      return false;
+    }
+    ac++;
+  }
+  if (ac != Arity()) {
+    cerr << "[ERROR] e-side arity mismatches f-side\n";
+    return false;
+  }
+  return true;
+}
+
+void TRule::ComputeArity() {
+  int min = 1;
+  for (vector<WordID>::const_iterator i = e_.begin(); i != e_.end(); ++i)
+    if (*i < min) min = *i;
+  arity_ = 1 - min;
+}
+
+static string AnonymousStrVar(int i) {
+  string res("[v]");
+  if(!(i <= 0 && i >= -8)) {
+    cerr << "Can't handle more than 9 non-terminals: index=" << (-i) << endl;
+    abort();
+  }
+  res[1] = '1' - i;
+  return res;
+}
+
+string TRule::AsString(bool verbose) const {
+  ostringstream os;
+  int idx = 0;
+  if (lhs_ && verbose) {
+    os << '[' << TD::Convert(lhs_ * -1) << "] |||";
+    for (int i = 0; i < f_.size(); ++i) {
+      const WordID& w = f_[i];
+      if (w < 0) {
+        int wi = w * -1;
+        ++idx;
+        os << " [" << TD::Convert(wi) << ',' << idx << ']';
+      } else {
+        os << ' ' << TD::Convert(w);
+      }
+    }
+    os << " ||| ";
+  }
+  if (idx > 9) {
+    cerr << "Too many non-terminals!\n partial: " << os.str() << endl;
+    exit(1);
+  }
+  for (int i =0; i<e_.size(); ++i) {
+    if (i) os << ' ';
+    const WordID& w = e_[i];
+    if (w < 1)
+      os << AnonymousStrVar(w);
+    else
+      os << TD::Convert(w);
+  }
+  if (!scores_.empty() && verbose) {
+    os << " ||| " << scores_;
+  }
+  return os.str();
+}
diff --git a/decoder/trule.h b/decoder/trule.h
new file mode 100644
index 00000000..7fb92924
--- /dev/null
+++ b/decoder/trule.h
@@ -0,0 +1,145 @@
+#ifndef _RULE_H_
+#define _RULE_H_
+
+#include <algorithm>
+#include <vector>
+#include <cassert>
+#include <boost/shared_ptr.hpp>
+
+#include "sparse_vector.h"
+#include "wordid.h"
+
+class TRule;
+typedef boost::shared_ptr<TRule> TRulePtr;
+
+struct NTSizeSummaryStatistics {
+  NTSizeSummaryStatistics(int arity) : means(arity), vars(arity) {}
+  std::vector<float> means;
+  std::vector<float> vars;
+};
+
+// Translation rule
+class TRule {
+ public:
+  TRule() : lhs_(0), prev_i(-1), prev_j(-1) { }
+  TRule(WordID lhs, const WordID* src, int src_size, const WordID* trg, int trg_size, const int* feat_ids, const double* feat_vals, int feat_size, int arity) :
+      e_(trg, trg + trg_size), f_(src, src + src_size), lhs_(lhs), arity_(arity), prev_i(-1), prev_j(-1) {
+    for (int i = 0; i < feat_size; ++i)
+      scores_.set_value(feat_ids[i], feat_vals[i]);
+  }
+
+  explicit TRule(const std::vector<WordID>& e) : e_(e), lhs_(0), prev_i(-1), prev_j(-1) {}
+  TRule(const std::vector<WordID>& e, const std::vector<WordID>& f, const WordID& lhs) :
+    e_(e), f_(f), lhs_(lhs), prev_i(-1), prev_j(-1) {}
+
+  // deprecated - this will be private soon
+  explicit TRule(const std::string& text, bool strict = false, bool mono = false) : prev_i(-1), prev_j(-1) {
+    ReadFromString(text, strict, mono);
+  }
+
+  // deprecated, use lexer
+  // make a rule from a hiero-like rule table, e.g.
+  //    [X] ||| [X,1] DE [X,2] ||| [X,2] of the [X,1]
+  // if misformatted, returns NULL
+  static TRule* CreateRuleSynchronous(const std::string& rule);
+
+  // deprecated, use lexer
+  // make a rule from a phrasetable entry (i.e., one that has no LHS type), e.g:
+  //    el gato ||| the cat ||| Feature_2=0.34
+  static TRule* CreateRulePhrasetable(const std::string& rule);
+
+  // deprecated, use lexer
+  // make a rule from a non-synchrnous CFG representation, e.g.:
+  //    [LHS] ||| term1 [NT] term2 [OTHER_NT] [YET_ANOTHER_NT]
+  static TRule* CreateRuleMonolingual(const std::string& rule);
+
+  static TRule* CreateLexicalRule(const WordID& src, const WordID& trg) {
+    return new TRule(src, trg);
+  }
+
+  void ESubstitute(const std::vector<const std::vector<WordID>* >& var_values,
+                   std::vector<WordID>* result) const {
+    int vc = 0;
+    result->clear();
+    for (std::vector<WordID>::const_iterator i = e_.begin(); i != e_.end(); ++i) {
+      const WordID& c = *i;
+      if (c < 1) {
+        ++vc;
+        const std::vector<WordID>& var_value = *var_values[-c];
+        std::copy(var_value.begin(),
+                  var_value.end(),
+                  std::back_inserter(*result));
+      } else {
+        result->push_back(c);
+      }
+    }
+    assert(vc == var_values.size());
+  }
+
+  void FSubstitute(const std::vector<const std::vector<WordID>* >& var_values,
+                   std::vector<WordID>* result) const {
+    int vc = 0;
+    result->clear();
+    for (std::vector<WordID>::const_iterator i = f_.begin(); i != f_.end(); ++i) {
+      const WordID& c = *i;
+      if (c < 1) {
+        const std::vector<WordID>& var_value = *var_values[vc++];
+        std::copy(var_value.begin(),
+                  var_value.end(),
+                  std::back_inserter(*result));
+      } else {
+        result->push_back(c);
+      }
+    }
+    assert(vc == var_values.size());
+  }
+
+  bool ReadFromString(const std::string& line, bool strict = false, bool monolingual = false);
+
+  bool Initialized() const { return e_.size(); }
+
+  std::string AsString(bool verbose = true) const;
+
+  static TRule DummyRule() {
+    TRule res;
+    res.e_.resize(1, 0);
+    return res;
+  }
+
+  const std::vector<WordID>& f() const { return f_; }
+  const std::vector<WordID>& e() const { return e_; }
+
+  int EWords() const { return ELength() - Arity(); }
+  int FWords() const { return FLength() - Arity(); }
+  int FLength() const { return f_.size(); }
+  int ELength() const { return e_.size(); }
+  int Arity() const { return arity_; }
+  bool IsUnary() const { return (Arity() == 1) && (f_.size() == 1); }
+  const SparseVector<double>& GetFeatureValues() const { return scores_; }
+  double Score(int i) const { return scores_[i]; }
+  WordID GetLHS() const { return lhs_; }
+  void ComputeArity();
+
+  // 0 = first variable, -1 = second variable, -2 = third ...
+  std::vector<WordID> e_;
+  // < 0: *-1 = encoding of category of variable
+  std::vector<WordID> f_;
+  WordID lhs_;
+  SparseVector<double> scores_;
+  
+  char arity_;
+  TRulePtr parent_rule_;  // usually NULL, except when doing constrained decoding
+
+  // this is only used when doing synchronous parsing
+  short int prev_i;
+  short int prev_j;
+
+  // may be null
+  boost::shared_ptr<NTSizeSummaryStatistics> nt_size_summary_;
+
+ private:
+  TRule(const WordID& src, const WordID& trg) : e_(1, trg), f_(1, src), lhs_(), arity_(), prev_i(), prev_j() {}
+  bool SanityCheck() const;
+};
+
+#endif
diff --git a/decoder/trule_test.cc b/decoder/trule_test.cc
new file mode 100644
index 00000000..02a70764
--- /dev/null
+++ b/decoder/trule_test.cc
@@ -0,0 +1,65 @@
+#include "trule.h"
+
+#include <gtest/gtest.h>
+#include <cassert>
+#include <iostream>
+#include "tdict.h"
+
+using namespace std;
+
+class TRuleTest : public testing::Test {
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+};
+
+TEST_F(TRuleTest,TestFSubstitute) {
+  TRule r1("[X] ||| ob [X,1] [X,2] sah . ||| whether [X,1] saw [X,2] . ||| 0.99");
+  TRule r2("[X] ||| ich ||| i ||| 1.0");
+  TRule r3("[X] ||| ihn ||| him ||| 1.0");
+  vector<const vector<WordID>*> ants;
+  vector<WordID> res2;
+  r2.FSubstitute(ants, &res2);
+  assert(TD::GetString(res2) == "ich");
+  vector<WordID> res3;
+  r3.FSubstitute(ants, &res3);
+  assert(TD::GetString(res3) == "ihn");
+  ants.push_back(&res2);
+  ants.push_back(&res3);
+  vector<WordID> res;
+  r1.FSubstitute(ants, &res);
+  cerr << TD::GetString(res) << endl;
+  assert(TD::GetString(res) == "ob ich ihn sah .");
+}
+
+TEST_F(TRuleTest,TestPhrasetableRule) {
+  TRulePtr t(TRule::CreateRulePhrasetable("gato ||| cat ||| PhraseModel_0=-23.2;Foo=1;Bar=12"));
+  cerr << t->AsString() << endl;
+  assert(t->scores_.num_active() == 3);
+};
+
+
+TEST_F(TRuleTest,TestMonoRule) {
+  TRulePtr m(TRule::CreateRuleMonolingual("[LHS] ||| term1 [NT] term2 [NT2] [NT3]"));
+  assert(m->Arity() == 3);
+  cerr << m->AsString() << endl;
+  TRulePtr m2(TRule::CreateRuleMonolingual("[LHS] ||| term1 [NT] term2 [NT2] [NT3] ||| Feature1=0.23"));
+  assert(m2->Arity() == 3);
+  cerr << m2->AsString() << endl;
+  EXPECT_FLOAT_EQ(m2->scores_.value(FD::Convert("Feature1")), 0.23);
+}
+
+TEST_F(TRuleTest,TestRuleR) {
+  TRule t6;
+  t6.ReadFromString("[X] ||| den [X,1] sah [X,2] . ||| [X,2] saw the [X,1] . ||| 0.12321 0.23232 0.121");
+  cerr << "TEXT: " << t6.AsString() << endl;
+  EXPECT_EQ(t6.Arity(), 2);
+  EXPECT_EQ(t6.e_[0], -1);
+  EXPECT_EQ(t6.e_[3], 0);
+}
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
diff --git a/decoder/ttables.cc b/decoder/ttables.cc
new file mode 100644
index 00000000..2ea960f0
--- /dev/null
+++ b/decoder/ttables.cc
@@ -0,0 +1,31 @@
+#include "ttables.h"
+
+#include <cassert>
+
+#include "dict.h"
+
+using namespace std;
+using namespace std::tr1;
+
+void TTable::DeserializeProbsFromText(std::istream* in) {
+  int c = 0;
+  while(*in) {
+    string e;
+    string f;
+    double p;
+    (*in) >> e >> f >> p;
+    if (e.empty()) break;
+    ++c;
+    ttable[TD::Convert(e)][TD::Convert(f)] = prob_t(p);
+  }
+  cerr << "Loaded " << c << " translation parameters.\n";
+}
+
+void TTable::SerializeHelper(string* out, const Word2Word2Double& o) {
+  assert(!"not implemented");
+}
+
+void TTable::DeserializeHelper(const string& in, Word2Word2Double* o) {
+  assert(!"not implemented");
+}
+
diff --git a/decoder/ttables.h b/decoder/ttables.h
new file mode 100644
index 00000000..3ffc238a
--- /dev/null
+++ b/decoder/ttables.h
@@ -0,0 +1,87 @@
+#ifndef _TTABLES_H_
+#define _TTABLES_H_
+
+#include <iostream>
+#include <map>
+
+#include "wordid.h"
+#include "prob.h"
+#include "tdict.h"
+
+class TTable {
+ public:
+  TTable() {}
+  typedef std::map<WordID, double> Word2Double;
+  typedef std::map<WordID, Word2Double> Word2Word2Double;
+  inline const prob_t prob(const int& e, const int& f) const {
+    const Word2Word2Double::const_iterator cit = ttable.find(e);
+    if (cit != ttable.end()) {
+      const Word2Double& cpd = cit->second;
+      const Word2Double::const_iterator it = cpd.find(f);
+      if (it == cpd.end()) return prob_t(0.00001);
+      return prob_t(it->second);
+    } else {
+      return prob_t(0.00001);
+    }
+  }
+  inline void Increment(const int& e, const int& f) {
+    counts[e][f] += 1.0;
+  }
+  inline void Increment(const int& e, const int& f, double x) {
+    counts[e][f] += x;
+  }
+  void Normalize() {
+    ttable.swap(counts);
+    for (Word2Word2Double::iterator cit = ttable.begin();
+         cit != ttable.end(); ++cit) {
+      double tot = 0;
+      Word2Double& cpd = cit->second;
+      for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it)
+        tot += it->second;
+      for (Word2Double::iterator it = cpd.begin(); it != cpd.end(); ++it)
+        it->second /= tot;
+    }
+    counts.clear();
+  }
+  // adds counts from another TTable - probabilities remain unchanged
+  TTable& operator+=(const TTable& rhs) {
+    for (Word2Word2Double::const_iterator it = rhs.counts.begin();
+         it != rhs.counts.end(); ++it) {
+      const Word2Double& cpd = it->second;
+      Word2Double& tgt = counts[it->first];
+      for (Word2Double::const_iterator j = cpd.begin(); j != cpd.end(); ++j) {
+        tgt[j->first] += j->second;
+      }
+    }
+    return *this;
+  }
+  void ShowTTable() {
+    for (Word2Word2Double::iterator it = ttable.begin(); it != ttable.end(); ++it) {
+      Word2Double& cpd = it->second;
+      for (Word2Double::iterator j = cpd.begin(); j != cpd.end(); ++j) {
+        std::cerr << "P(" << TD::Convert(j->first) << '|' << TD::Convert(it->first) << ") = " << j->second << std::endl;
+      }
+    }
+  }
+  void ShowCounts() {
+    for (Word2Word2Double::iterator it = counts.begin(); it != counts.end(); ++it) {
+      Word2Double& cpd = it->second;
+      for (Word2Double::iterator j = cpd.begin(); j != cpd.end(); ++j) {
+        std::cerr << "c(" << TD::Convert(j->first) << '|' << TD::Convert(it->first) << ") = " << j->second << std::endl;
+      }
+    }
+  }
+  void DeserializeProbsFromText(std::istream* in);
+  void SerializeCounts(std::string* out) const { SerializeHelper(out, counts); }
+  void DeserializeCounts(const std::string& in) { DeserializeHelper(in, &counts); }
+  void SerializeProbs(std::string* out) const { SerializeHelper(out, ttable); }
+  void DeserializeProbs(const std::string& in) { DeserializeHelper(in, &ttable); }
+ private:
+  static void SerializeHelper(std::string*, const Word2Word2Double& o);
+  static void DeserializeHelper(const std::string&, Word2Word2Double* o);
+ public:
+  Word2Word2Double ttable;
+  Word2Word2Double counts;
+};
+
+#endif
diff --git a/decoder/viterbi.cc b/decoder/viterbi.cc
new file mode 100644
index 00000000..82b2ce6d
--- /dev/null
+++ b/decoder/viterbi.cc
@@ -0,0 +1,39 @@
+#include "viterbi.h"
+
+#include <vector>
+#include "hg.h"
+
+using namespace std;
+
+string ViterbiETree(const Hypergraph& hg) {
+  vector<WordID> tmp;
+  const prob_t p = Viterbi<vector<WordID>, ETreeTraversal, prob_t, EdgeProb>(hg, &tmp);
+  return TD::GetString(tmp);
+}
+
+string ViterbiFTree(const Hypergraph& hg) {
+  vector<WordID> tmp;
+  const prob_t p = Viterbi<vector<WordID>, FTreeTraversal, prob_t, EdgeProb>(hg, &tmp);
+  return TD::GetString(tmp);
+}
+
+prob_t ViterbiESentence(const Hypergraph& hg, vector<WordID>* result) {
+  return Viterbi<vector<WordID>, ESentenceTraversal, prob_t, EdgeProb>(hg, result);
+}
+
+prob_t ViterbiFSentence(const Hypergraph& hg, vector<WordID>* result) {
+  return Viterbi<vector<WordID>, FSentenceTraversal, prob_t, EdgeProb>(hg, result);
+}
+
+int ViterbiELength(const Hypergraph& hg) {
+  int len = -1;
+  Viterbi<int, ELengthTraversal, prob_t, EdgeProb>(hg, &len);
+  return len;
+}
+
+int ViterbiPathLength(const Hypergraph& hg) {
+  int len = -1;
+  Viterbi<int, PathLengthTraversal, prob_t, EdgeProb>(hg, &len);
+  return len;
+}
+
diff --git a/decoder/viterbi.h b/decoder/viterbi.h
new file mode 100644
index 00000000..8f7534a9
--- /dev/null
+++ b/decoder/viterbi.h
@@ -0,0 +1,142 @@
+#ifndef _VITERBI_H_
+#define _VITERBI_H_
+
+#include <vector>
+#include "prob.h"
+#include "hg.h"
+#include "tdict.h"
+
+// V must implement:
+//  void operator()(const vector<const T*>& ants, T* result);
+template<typename T, typename Traversal, typename WeightType, typename WeightFunction>
+WeightType Viterbi(const Hypergraph& hg,
+                   T* result,
+                   const Traversal& traverse = Traversal(),
+                   const WeightFunction& weight = WeightFunction()) {
+  const int num_nodes = hg.nodes_.size();
+  std::vector<T> vit_result(num_nodes);
+  std::vector<WeightType> vit_weight(num_nodes, WeightType::Zero());
+
+  for (int i = 0; i < num_nodes; ++i) {
+    const Hypergraph::Node& cur_node = hg.nodes_[i];
+    WeightType* const cur_node_best_weight = &vit_weight[i];
+    T*          const cur_node_best_result = &vit_result[i];
+    
+    const int num_in_edges = cur_node.in_edges_.size();
+    if (num_in_edges == 0) {
+      *cur_node_best_weight = WeightType(1);
+      continue;
+    }
+    for (int j = 0; j < num_in_edges; ++j) {
+      const Hypergraph::Edge& edge = hg.edges_[cur_node.in_edges_[j]];
+      WeightType score = weight(edge);
+      std::vector<const T*> ants(edge.tail_nodes_.size());
+      for (int k = 0; k < edge.tail_nodes_.size(); ++k) {
+        const int tail_node_index = edge.tail_nodes_[k];
+        score *= vit_weight[tail_node_index];
+        ants[k] = &vit_result[tail_node_index];
+      }
+      if (*cur_node_best_weight < score) {
+        *cur_node_best_weight = score;
+        traverse(edge, ants, cur_node_best_result);
+      }
+    }
+  }
+  std::swap(*result, vit_result.back());
+  return vit_weight.back();
+}
+
+struct PathLengthTraversal {
+  void operator()(const Hypergraph::Edge& edge,
+                  const std::vector<const int*>& ants,
+                  int* result) const {
+    (void) edge;
+    *result = 1;
+    for (int i = 0; i < ants.size(); ++i) *result += *ants[i];
+  }
+};
+
+struct ESentenceTraversal {
+  void operator()(const Hypergraph::Edge& edge,
+                  const std::vector<const std::vector<WordID>*>& ants,
+                  std::vector<WordID>* result) const {
+    edge.rule_->ESubstitute(ants, result);
+  }
+};
+
+struct ELengthTraversal {
+  void operator()(const Hypergraph::Edge& edge,
+                  const std::vector<const int*>& ants,
+                  int* result) const {
+    *result = edge.rule_->ELength() - edge.rule_->Arity();
+    for (int i = 0; i < ants.size(); ++i) *result += *ants[i];
+  }
+};
+
+struct FSentenceTraversal {
+  void operator()(const Hypergraph::Edge& edge,
+                  const std::vector<const std::vector<WordID>*>& ants,
+                  std::vector<WordID>* result) const {
+    edge.rule_->FSubstitute(ants, result);
+  }
+};
+
+// create a strings of the form (S (X the man) (X said (X he (X would (X go)))))
+struct ETreeTraversal {
+  ETreeTraversal() : left("("), space(" "), right(")") {}
+  const std::string left;
+  const std::string space;
+  const std::string right;
+  void operator()(const Hypergraph::Edge& edge,
+                  const std::vector<const std::vector<WordID>*>& ants,
+                  std::vector<WordID>* result) const {
+    std::vector<WordID> tmp;
+    edge.rule_->ESubstitute(ants, &tmp);
+    const std::string cat = TD::Convert(edge.rule_->GetLHS() * -1);
+    if (cat == "Goal")
+      result->swap(tmp);
+    else
+      TD::ConvertSentence(left + cat + space + TD::GetString(tmp) + right,
+                          result);
+  }
+};
+
+struct FTreeTraversal {
+  FTreeTraversal() : left("("), space(" "), right(")") {}
+  const std::string left;
+  const std::string space;
+  const std::string right;
+  void operator()(const Hypergraph::Edge& edge,
+                  const std::vector<const std::vector<WordID>*>& ants,
+                  std::vector<WordID>* result) const {
+    std::vector<WordID> tmp;
+    edge.rule_->FSubstitute(ants, &tmp);
+    const std::string cat = TD::Convert(edge.rule_->GetLHS() * -1);
+    if (cat == "Goal")
+      result->swap(tmp);
+    else
+      TD::ConvertSentence(left + cat + space + TD::GetString(tmp) + right,
+                          result);
+  }
+};
+
+struct ViterbiPathTraversal {
+  void operator()(const Hypergraph::Edge& edge,
+                  const std::vector<const std::vector<const Hypergraph::Edge*>* >& ants,
+                  std::vector<const Hypergraph::Edge*>* result) const {
+    result->clear();
+    for (int i = 0; i < ants.size(); ++i)
+      for (int j = 0; j < ants[i]->size(); ++j)
+        result->push_back((*ants[i])[j]);
+    result->push_back(&edge);
+  }
+};
+
+prob_t ViterbiESentence(const Hypergraph& hg, std::vector<WordID>* result);
+std::string ViterbiETree(const Hypergraph& hg);
+prob_t ViterbiFSentence(const Hypergraph& hg, std::vector<WordID>* result);
+std::string ViterbiFTree(const Hypergraph& hg);
+int ViterbiELength(const Hypergraph& hg);
+int ViterbiPathLength(const Hypergraph& hg);
+
+#endif
diff --git a/decoder/weights.cc b/decoder/weights.cc
new file mode 100644
index 00000000..84647585
--- /dev/null
+++ b/decoder/weights.cc
@@ -0,0 +1,77 @@
+#include "weights.h"
+
+#include <sstream>
+
+#include "fdict.h"
+#include "filelib.h"
+
+using namespace std;
+
+void Weights::InitFromFile(const std::string& filename, vector<string>* feature_list) {
+  cerr << "Reading weights from " << filename << endl;
+  ReadFile in_file(filename);
+  istream& in = *in_file.stream();
+  assert(in);
+  int weight_count = 0;
+  bool fl = false;
+  while (in) {
+    double val = 0;
+    string buf;
+    getline(in, buf);
+    if (buf.size() == 0) continue;
+    if (buf[0] == '#') continue;
+    for (int i = 0; i < buf.size(); ++i)
+      if (buf[i] == '=') buf[i] = ' ';
+    int start = 0;
+    while(start < buf.size() && buf[start] == ' ') ++start;
+    int end = 0;
+    while(end < buf.size() && buf[end] != ' ') ++end;
+    int fid = FD::Convert(buf.substr(start, end - start));
+    while(end < buf.size() && buf[end] == ' ') ++end;
+    val = strtod(&buf.c_str()[end], NULL);
+    if (isnan(val)) {
+      cerr << FD::Convert(fid) << " has weight NaN!\n";
+      abort();
+    }
+    if (wv_.size() <= fid)
+      wv_.resize(fid + 1);
+    wv_[fid] = val;
+    if (feature_list) { feature_list->push_back(FD::Convert(fid)); }
+    ++weight_count;
+    if (weight_count %   50000 == 0) { cerr << '.' << flush; fl = true; }
+    if (weight_count % 2000000 == 0) { cerr << " [" << weight_count << "]\n"; fl = false; }
+  }
+  if (fl) { cerr << endl; }
+  cerr << "Loaded " << weight_count << " feature weights\n";
+}
+
+void Weights::WriteToFile(const std::string& fname, bool hide_zero_value_features) const {
+  WriteFile out(fname);
+  ostream& o = *out.stream();
+  assert(o);
+  o.precision(17);
+  const int num_feats = FD::NumFeats();
+  for (int i = 1; i < num_feats; ++i) {
+    const double val = (i < wv_.size() ? wv_[i] : 0.0);
+    if (hide_zero_value_features && val == 0.0) continue;
+    o << FD::Convert(i) << ' ' << val << endl;
+  }
+}
+
+void Weights::InitVector(std::vector<double>* w) const {
+  *w = wv_;
+}
+
+void Weights::InitSparseVector(SparseVector<double>* w) const {
+  for (int i = 1; i < wv_.size(); ++i) {
+    const double& weight = wv_[i];
+    if (weight) w->set_value(i, weight);
+  }
+}
+
+void Weights::InitFromVector(const std::vector<double>& w) {
+  wv_ = w;
+  if (wv_.size() > FD::NumFeats())
+    cerr << "WARNING: initializing weight vector has more features than the global feature dictionary!\n";
+  wv_.resize(FD::NumFeats(), 0);
+}
diff --git a/decoder/weights.h b/decoder/weights.h
new file mode 100644
index 00000000..f19aa3ce
--- /dev/null
+++ b/decoder/weights.h
@@ -0,0 +1,21 @@
+#ifndef _WEIGHTS_H_
+#define _WEIGHTS_H_
+
+#include <string>
+#include <map>
+#include <vector>
+#include "sparse_vector.h"
+
+class Weights {
+ public:
+  Weights() {}
+  void InitFromFile(const std::string& fname, std::vector<std::string>* feature_list = NULL);
+  void WriteToFile(const std::string& fname, bool hide_zero_value_features = true) const;
+  void InitVector(std::vector<double>* w) const;
+  void InitSparseVector(SparseVector<double>* w) const;
+  void InitFromVector(const std::vector<double>& w);
+ private:
+  std::vector<double> wv_;
+};
+
+#endif
diff --git a/decoder/weights_test.cc b/decoder/weights_test.cc
new file mode 100644
index 00000000..aa6b3db2
--- /dev/null
+++ b/decoder/weights_test.cc
@@ -0,0 +1,28 @@
+#include <cassert>
+#include <iostream>
+#include <fstream>
+#include <vector>
+#include <gtest/gtest.h>
+#include "weights.h"
+#include "tdict.h"
+#include "hg.h"
+
+using namespace std;
+
+class WeightsTest : public testing::Test {
+ protected:
+  virtual void SetUp() { }
+  virtual void TearDown() { }
+};
+       
+
+TEST_F(WeightsTest,Load) {
+  Weights w;
+  w.InitFromFile("test_data/weights");
+  w.WriteToFile("-");
+}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/decoder/wordid.h b/decoder/wordid.h
new file mode 100644
index 00000000..fb50bcc1
--- /dev/null
+++ b/decoder/wordid.h
@@ -0,0 +1,6 @@
+#ifndef _WORD_ID_H_
+#define _WORD_ID_H_
+
+typedef int WordID;
+
+#endif