From ecccfb93c2e259fa5b6bb672cb9cadfa68d573a6 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Sat, 18 Jan 2014 21:57:32 -0500
Subject: new tuning of crf compound splitter for wmt14

---
 decoder/ff_csplit.cc | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'decoder')

diff --git a/decoder/ff_csplit.cc b/decoder/ff_csplit.cc
index 33b6cea8..a0e538d3 100644
--- a/decoder/ff_csplit.cc
+++ b/decoder/ff_csplit.cc
@@ -37,7 +37,8 @@ struct BasicCSplitFeaturesImpl {
       short_range_(FD::Convert("ShortRange")),
       high_freq_(FD::Convert("HighFreq")),
       med_freq_(FD::Convert("MedFreq")),
-      freq_(FD::Convert("Freq")),
+      logfreq_(FD::Convert("LogFreq")),
+      loglogfreq_(FD::Convert("LogLogFreq")),
       in_dict_full_word_(FD::Convert("InDictFullWord")),
       fl1_(FD::Convert("FreqLen1")),
       fl2_(FD::Convert("FreqLen2")),
@@ -86,7 +87,8 @@ struct BasicCSplitFeaturesImpl {
   const int short_range_;
   const int high_freq_;
   const int med_freq_;
-  const int freq_;
+  const int logfreq_;
+  const int loglogfreq_;
   const int in_dict_full_word_;
   const int fl1_;
   const int fl2_;
@@ -139,7 +141,8 @@ void BasicCSplitFeaturesImpl::TraversalFeaturesImpl(
 
   float freq = freq_dict_.LookUp(word);
   if (freq) {
-    features->set_value(freq_, freq);
+    features->set_value(logfreq_, freq);
+    features->set_value(loglogfreq_, log(freq) / log(1.69));
     features->set_value(in_dict_, 1.0);
     if (subword) features->set_value(in_dict_sub_word_, 1.0);
   } else {
-- 
cgit v1.2.3