add compound splitting logic and features (Dyer 2008, NAACL)

1 files changed, 10 insertions, 4 deletions

diff --git a/src/freqdict.cc b/src/freqdict.cc
index 4cfffe58..9e25d346 100644
--- a/src/freqdict.cc
+++ b/src/freqdict.cc
@@ -2,11 +2,17 @@
 #include <fstream>
 #include <cassert>
 #include "freqdict.h"
+#include "tdict.h"
+#include "filelib.h"
 
-void FreqDict::load(const std::string& fname) {
-  std::ifstream ifs(fname.c_str());
+using namespace std;
+
+void FreqDict::Load(const std::string& fname) {
+  cerr << "Reading word frequencies: " << fname << endl;
+  ReadFile rf(fname);
+  istream& ifs = *rf.stream();
   int cc=0;
-  while (!ifs.eof()) {
+  while (ifs) {
     std::string word;
     ifs >> word;
     if (word.size() == 0) continue;
@@ -14,7 +20,7 @@ void FreqDict::load(const std::string& fname) {
     double count = 0;
     ifs >> count;
     assert(count > 0.0);  // use -log(f)
-    counts_[word]=count;
+    counts_[TD::Convert(word)]=count;
     ++cc;
     if (cc % 10000 == 0) { std::cerr << "."; }
   }