From 22f37b828e2e9d8fc2cb9a7f55081b5a327f709f Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@Chriss-MacBook-Air.local>
Date: Mon, 6 May 2013 22:18:08 -0400
Subject: add passthrough length features

---
 decoder/scfg_translator.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'decoder')

diff --git a/decoder/scfg_translator.cc b/decoder/scfg_translator.cc
index 3b43b586..6f0b003b 100644
--- a/decoder/scfg_translator.cc
+++ b/decoder/scfg_translator.cc
@@ -12,6 +12,7 @@
 #include "grammar.h"
 #include "bottom_up_parser.h"
 #include "sentence_metadata.h"
+#include "stringlib.h"
 #include "tdict.h"
 #include "viterbi.h"
 #include "verbose.h"
@@ -68,7 +69,11 @@ PassThroughGrammar::PassThroughGrammar(const Lattice& input, const string& cat,
       const int j = alts[k].dist2next + i;
       const string& src = TD::Convert(alts[k].label);
       if (ss.count(alts[k].label) == 0) {
-        TRulePtr pt(new TRule("[" + cat + "] ||| " + src + " ||| " + src + " ||| PassThrough=1"));
+        int length = static_cast<int>(log(UTF8StringLen(src)) / log(1.6)) + 1;
+        if (length > 6) length = 6;
+        string len_feat = "PassThrough_0=1";
+        len_feat[12] += length;
+        TRulePtr pt(new TRule("[" + cat + "] ||| " + src + " ||| " + src + " ||| PassThrough=1 " + len_feat));
         pt->a_.push_back(AlignmentPoint(0,0));
         AddRule(pt);
         RefineRule(pt, ctf_level);
-- 
cgit v1.2.3