Merge branch 'master' of https://github.com/redpony/cdec

1 files changed, 0 insertions, 894 deletions

diff --git a/decoder/ff_dwarf.cc b/decoder/ff_dwarf.cc
deleted file mode 100644
index fe7a472e..00000000
--- a/decoder/ff_dwarf.cc
+++ /dev/null
@@ -1,894 +0,0 @@
-#include <vector>
-#include <sstream>
-#include <fstream>
-#include <string>
-#include <iostream>
-#include <map>
-#include "hg.h"
-#include "ff_dwarf.h"
-#include "dwarf.h"
-#include "wordid.h"
-#include "tdict.h"
-#include "filelib.h"
-#include "sentence_metadata.h"
-#include "stringlib.h"
-
-using namespace std;
-
-Dwarf::Dwarf(const std::string& param) { 
-/* Param is a space separated string which contains any or all of the following:
-   oris|orit|doms|domt=filename 
-   e.g. oris=/fs/clip-galep3eval/hendra/z2e/oris128.gz
-*/
-  sSOS="<s>";
-  sEOS="</s>";
-  kSOS=TD::Convert(sSOS);
-  kEOS=TD::Convert(sEOS);
-  kGOAL=TD::Convert("S")*-1;
-  _sent_id = (int *)malloc(sizeof(int));
-  *_sent_id = -1;
-  if (DEBUG) cerr << "here = " << *_sent_id << endl;
-  _fwcount = (int *)malloc(sizeof(int));
-  *_fwcount = -1;
-  cerr << "initializing dwarf" << endl;
-  flag_oris=false; flag_orit=false; flag_doms=false; flag_domt=false; flag_tfw_count=false;
-  flag_bdoms=false; flag_porislr=false, flag_porisrl=false, flag_goris=false; flag_pgorislr=false, flag_pgorisrl=false;
-  flag_pdomslr=false; flag_pdomsrl=false; flag_pgdomslr=false; flag_pgdomsrl=false; flag_gdoms=false;
-  flag_oris_backward=false; flag_orit_backward=false; 
-  explicit_soseos=false;
-  SetStateSize(STATE_SIZE*sizeof(int));   
-  als = new Alignment();  
-  als->clearAls(Alignment::MAX_WORDS,Alignment::MAX_WORDS);
-  istringstream iss(param); string w;
-  while(iss >> w) {
-    int equal = w.find_first_of("=");
-    if (equal!=string::npos) {
-      string model = w.substr(0,equal);
-      vector<string> params; 
-      Tokenize(w.substr(equal+1),',',&params);
-      string fn = params[0];
-      if (model == "minfreq") {
-        cerr << "model minfreq " << fn << endl;
-        als->setFreqCutoff(atoi(fn.c_str()));
-      } else if (model == "oris") {
-        flag_oris = readOrientation(&toris,fn,&sfw); 
-        if (flag_oris) {
-          oris_ = FD::Convert("OrientationSource");
-          //oris_bo1_ = FD::Convert("OrientationSource_BO1");
-          //oris_bo2_ = FD::Convert("OrientationSource_BO2");
-        }
-        if (params.size()>1) als->setAlphaOris(atof(params[1].c_str()));
-        if (params.size()>2) als->setBetaOris(atof(params[2].c_str()));
-      } else if (model == "porislr") {
-        flag_porislr = readOrientation(&tporislr,fn,&sfw,true);
-        poris_nlr = 0;
-        if (flag_porislr) {
-          porislr_ = FD::Convert("OrientationSourcePositionfulLeftRight");
-        }
-        if (params.size()>1) poris_nlr = atoi(params[1].c_str());
-        if (DEBUG) cerr << "  maximum poris depth=" << poris_nlr  << endl;
-      } else if (model == "porisrl") {
-        flag_porisrl = readOrientation(&tporisrl,fn,&sfw,true);
-        poris_nrl = 0;
-        if (flag_porisrl) {
-          porisrl_ = FD::Convert("OrientationSourcePositionfulRightLeft");
-        }
-        if (params.size()>1) poris_nrl = atoi(params[1].c_str());
-        if (DEBUG) cerr << "  maximum poris depth=" << poris_nrl  << endl;
-      } else if (model=="goris") {
-        flag_goris = readOrientation(&tgoris,fn,&sfw);
-        if (flag_goris) {
-          goris_ = FD::Convert("OrientationSourceGeneralized");
-        }
-        if (params.size()>1) {
-          readTags(params[1],&tags);
-          generalizeOrientation(&tgoris,tags);          
-        }
-      } else if (model=="pgorislr") {
-        flag_pgorislr = readOrientation(&tpgorislr,fn,&sfw,true);
-        pgoris_nlr = 0;
-        if (flag_pgorislr) {
-          pgorislr_ = FD::Convert("OrientationSourceGeneralizedPositionfulLeftRight");
-        }
-        if (DEBUG) {
-          cerr << "BEFORE GENERALIZATION" << endl;
-          tpgorislr.print();
-        }
-        if (params.size()>1) pgoris_nlr = atoi(params[1].c_str());
-        if (params.size()>2) {
-          readTags(params[2],&tags);
-          generalizeOrientation(&tpgorislr,tags,true);
-        }
-        if (DEBUG) {
-          cerr << "AFTER GENERALIZATION" << endl;
-          tpgorislr.print();
-        }
-      } else if (model=="pgorisrl") {
-        flag_pgorisrl = readOrientation(&tpgorisrl,fn,&sfw,true);
-        pgoris_nrl = 0;
-        if (flag_pgorisrl) {
-          pgorisrl_ = FD::Convert("OrientationSourceGeneralizedPositionfulLeftRight");
-        } 
-        if (params.size()>1) pgoris_nrl = atoi(params[1].c_str());
-        if (params.size()>2) {
-          readTags(params[2],&tags);
-          generalizeOrientation(&tpgorisrl,tags,true);
-        }
-      } else if (model == "oris_backward") {
-        flag_oris_backward = true;
-        if (!flag_oris) readOrientation(&toris,fn,&sfw);
-        oris_backward_ = FD::Convert("OrientationSourceBackward");
-        if (params.size()>1) als->setAlphaOris(atof(params[1].c_str()));
-        if (params.size()>2) als->setBetaOris(atof(params[2].c_str()));
-      } else if (model == "orit") {
-        flag_orit = readOrientation(&torit,fn,&tfw); 
-        if (flag_orit) {
-          orit_ = FD::Convert("OrientationTarget");
-          //orit_bo1_ = FD::Convert("OrientationTarget_BO1");
-          //orit_bo2_ = FD::Convert("OrientationTarget_BO2");
-        }
-        if (params.size()>1) als->setAlphaOrit(atof(params[1].c_str()));
-        if (params.size()>2) als->setBetaOrit(atof(params[2].c_str()));
-      } else if (model == "orit_backward") {
-        flag_orit_backward = true;
-        if (!flag_orit) readOrientation(&torit,fn,&tfw);
-        orit_backward_ = FD::Convert("OrientationTargetBackward");
-        if (params.size()>1) als->setAlphaOrit(atof(params[1].c_str()));
-        if (params.size()>2) als->setBetaOrit(atof(params[2].c_str()));
-      } else if (model == "doms") {
-        flag_doms = readDominance(&tdoms,fn,&sfw); 
-        if (flag_doms) {
-          doms_ = FD::Convert("DominanceSource");
-          //doms_bo1_ = FD::Convert("DominanceSource_BO1");
-          //doms_bo2_ = FD::Convert("DominanceSource_BO2");
-        }
-        if (params.size()>1) als->setAlphaDoms(atof(params[1].c_str()));
-        if (params.size()>2) als->setBetaDoms(atof(params[2].c_str()));
-      } else if (model == "pdomsrl") {
-        flag_pdomsrl = readDominance(&tpdomsrl,fn,&sfw,true);
-        if (flag_pdomsrl) {
-          pdomsrl_ = FD::Convert("DominanceSourcePositionfulRightLeft");
-        }
-        if (params.size()>1) pdoms_nrl = atoi(params[1].c_str());
-      } else if (model == "pdomslr") {
-        flag_pdomslr = readDominance(&tpdomslr,fn,&sfw,true);
-        tpdomslr.print();
-        if (flag_pdomslr) {
-          pdomslr_ = FD::Convert("DominanceSourcePositionfulLeftRight");
-        }
-        if (params.size()>1) pdoms_nlr = atoi(params[1].c_str());
-      } else if (model == "pgdomsrl") {
-        flag_pgdomsrl = readDominance(&tpgdomsrl,fn,&sfw,true);
-        if (flag_pgdomsrl) {
-          pgdomsrl_ = FD::Convert("DominanceSourceGeneralizedPositionfulRightLeft");
-        }
-        if (params.size()>1) pgdoms_nrl = atoi(params[1].c_str());
-        if (params.size()>2) {
-          readTags(params[2],&tags);
-          generalizeDominance(&tpgdomsrl,tags,true);
-        }
-      } else if (model == "pgdomslr") {
-        flag_pgdomslr = readDominance(&tpgdomslr,fn,&sfw,true);
-        if (flag_pgdomslr) {
-          pgdomslr_ = FD::Convert("DominanceSourceGeneralizedPositionfulLeftRight");
-        }
-        if (params.size()>1) pgdoms_nlr = atoi(params[1].c_str());
-        if (params.size()>2) {
-          readTags(params[2],&tags);
-          if (DEBUG) {
-            for (map<WordID,WordID>::const_iterator it=tags.begin(); it!=tags.end(); it++) {
-              cerr << "tags = " << TD::Convert(it->first) << ", " << TD::Convert(it->second) << endl;
-            }
-          } 
-          generalizeDominance(&tpgdomslr,tags,true);
-        }
-        if (DEBUG) tpgdomslr.print();
-      } else if (model == "bdoms") {
-        flag_bdoms = readDominance(&tbdoms,fn,&sfw);
-        if (flag_bdoms) {
-          bdoms_ = FD::Convert("BorderDominanceSource");
-        }
-      } else if (model == "domt") {
-        flag_domt = readDominance(&tdomt,fn,&tfw); 
-        if (flag_domt) {
-          domt_ = FD::Convert("DominanceTarget");
-          //domt_bo1_ = FD::Convert("DominanceTarget_BO1");
-          //domt_bo2_ = FD::Convert("DominanceTarget_BO2");
-        }
-        if (params.size()>1) als->setAlphaDomt(atof(params[1].c_str()));
-        if (params.size()>2) als->setBetaDomt(atof(params[2].c_str()));
-      } else if (model== "tfw_count") {
-        flag_tfw_count = readList(fn,&tfw);
-        tfw_count_ = FD::Convert("TargetFunctionWordsCount");        
-      } else {
-        cerr << "DWARF doesn't understand this model: " << model << endl;
-      }
-    } else {
-      if (w=="tfw_count") {
-        flag_tfw_count = true;
-        tfw_count_ = FD::Convert("TargetFunctionWordsCount");
-      } else if (w=="oris_backward") {
-        flag_oris_backward = true;
-        oris_backward_ = FD::Convert("OrientationSourceBackward"); 
-      } else if (w=="orit_backward") {
-        flag_orit_backward = true;
-        orit_backward_ = FD::Convert("OrientationTargetBackward");
-      } else if (w=="explicit_soseos") {
-        explicit_soseos=true;
-      } else {
-        cerr << "DWARF doesn't need this param: " << param << endl; 
-      }
-    }  
-  }
-  for (map<WordID,int>::const_iterator it=sfw.begin(); it!=sfw.end() && DEBUG; it++) {
-    cerr << "   FW:" << TD::Convert(it->first) << endl;
-  }
-}
-
-void Dwarf::TraversalFeaturesImpl(const SentenceMetadata& smeta,
-                                     const Hypergraph::Edge& edge,
-                                     const std::vector<const void*>& ant_contexts,
-                                     SparseVector<double>* features,
-                                     SparseVector<double>* estimated_features,
-                                     void* context) const {
-  if (DEBUG) cerr << "TraversalFeaturesImpl" << endl;
-  double cost, bonus, bo1, bo2, bo1_bonus, bo2_bonus;
-  double bdoms_state_mono= 0; double bdoms_state_nonmono = 0;
-  TRule r = *edge.rule_;
-  if (DEBUG) cerr << " sent_id=" << *_sent_id << ", " << smeta.GetSentenceID() << endl;
-  if (DEBUG) cerr << "rule = " << r.AsString() << endl; 
-  if (DEBUG) cerr << "rule[i,j] = " << edge.i_ << "," << edge.j_ << endl;
-  if (*_sent_id != smeta.GetSentenceID()) { //new sentence
-    *_sent_id = smeta.GetSentenceID();
-    const Lattice l = smeta.GetSourceLattice();
-    *_fwcount=0;
-    for (int i=0; i<smeta.GetSourceLength(); i++) {
-      if (sfw.find(l[i][0].label)!=sfw.end()) {
-        *_fwcount+=1;
-      }
-    }
-    if (DEBUG) cerr << "new sentence[" << *_sent_id << "]="<<*_fwcount<<endl;
-  }
-  bool nofw = als->prepare(*edge.rule_, ant_contexts, sfw, tfw,smeta.GetSourceLattice(),edge.i_,edge.j_); 
-  bool isFinal = (edge.i_==0 && edge.j_==smeta.GetSourceLength() && r.GetLHS()==kGOAL);
-  // prepare *nofw* outputs whether the resulting alignment, contains function words or not
-  // if not, the models do not have to be calcualted and *simplify* is very simple
-  if (DEBUG) cerr << "nofw = " << nofw << endl;
-  if (flag_tfw_count) {
-    double count = 0;
-    for (int i=0; i<r.e_.size(); i++) {
-      if (tfw.find(r.e_[i])!=tfw.end()) count++;
-    }
-    features->set_value(tfw_count_,count);  
-  }
-  if (flag_oris) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw) als->computeOrientationSource(toris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); 
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(oris_,cost); 
-    //features->set_value(oris_bo1_,bo1); 
-    //features->set_value(oris_bo2_,bo2);
-    estimated_features->set_value(oris_,bonus); 
-    //estimated_features->set_value(oris_bo1_,bo1_bonus); 
-    //estimated_features->set_value(oris_bo2_,bo2_bonus);
-  }
-  if (flag_porislr) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw) 
-      als->computeOrientationSourcePos(tporislr,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,poris_nlr,0);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(porislr_,cost);
-    estimated_features->set_value(porislr_,bonus);
-  }
-  if (flag_porisrl) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw)
-      als->computeOrientationSourcePos(tporisrl,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,poris_nrl);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(porisrl_,cost);
-    estimated_features->set_value(porisrl_,bonus);
-  }
-  if (flag_pgorislr) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw)
-      als->computeOrientationSourcePos(tpgorislr,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pgoris_nlr,0);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(pgorislr_,cost);
-    estimated_features->set_value(pgorislr_,bonus);
-  }
-  if (flag_pgorisrl) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw)
-      als->computeOrientationSourcePos(tpgorisrl,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pgoris_nrl);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(pgorisrl_,cost);
-    estimated_features->set_value(pgorisrl_,bonus);
-  }
-  if (flag_goris) {
-    cost=0; bonus=0;
-    if (!nofw) als->computeOrientationSource(tgoris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(goris_,cost);
-    estimated_features->set_value(goris_,bonus);
-  }
-  if (flag_oris_backward) {
-    cost=0; bonus=0;
-    if (!nofw) 
-      als->computeOrientationSourceBackward(toris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(oris_backward_,cost);
-    estimated_features->set_value(oris_backward_,bonus);
-  }
-  WordID _lfw = kSOS;
-  WordID _rfw = kEOS; 
-  if (flag_doms || flag_pdomslr || flag_pdomsrl || flag_pgdomslr || flag_pgdomsrl) {
-    if (DEBUG) cerr << "   seeking lfw and rfw" << endl;
-    int start = edge.i_;
-    int end   = edge.j_;
-    if (DEBUG) cerr << "   start=" << start << ", end=" << end << endl;
-    const Lattice l = smeta.GetSourceLattice();
-    for (int idx=start-1; idx>=0; idx--) {
-      if (DEBUG) cerr << "  checking idx=" << idx << ", label=" << l[idx][0].label << "-" << TD::Convert(l[idx][0].label) << endl;
-      if (sfw.find(l[idx][0].label) !=sfw.end()) {
-        if (DEBUG) cerr << "+";
-        _lfw=l[idx][0].label; break;
-      }
-    }
-    for (int idx=end; idx<l.size(); idx++) { // end or end+1
-      if (DEBUG) cerr << "  checking idx=" << idx << ", label=" << l[idx][0].label << "-" << TD::Convert(l[idx][0].label) << endl;
-      if (sfw.find(l[idx][0].label)!=sfw.end()) {
-        if (DEBUG) cerr << ".";
-        _rfw=l[idx][0].label; break;
-      }
-    }
-    if (isFinal&&!explicit_soseos) {
-      _lfw=kSOS; _rfw=kEOS;
-    }
-  }
-  if (flag_doms) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw) als->computeDominanceSource(tdoms,_lfw,_rfw,&cost,&bonus,
-                                           &bo1,&bo1_bonus,&bo2,&bo2_bonus); 
-    if (DEBUG) cerr << "   COST=" << cost << ", BONUS=" << bonus << endl;
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      if (DEBUG) cerr << "    final and !explicit_soseos, thus cost = " << cost <<  endl;
-      bonus = 0;
-    }
-    features->set_value(doms_,cost); 
-    estimated_features->set_value(doms_,bonus);
-  }
-  if (flag_pdomslr) {
-   if (DEBUG) cerr << " flag_pdomslr true, nofw=" << nofw << endl;
-   if (DEBUG) cerr << "   lfw=" << _lfw << ", rfw=" << _rfw << endl;
-   if (DEBUG) cerr << "   kSOS=" << kSOS << ", kEOS=" << kEOS << endl;
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw) als->computeDominanceSourcePos(tpdomslr,_lfw,_rfw,&cost,&bonus,
-                                           &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pdoms_nlr,0);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(pdomslr_,cost);
-    estimated_features->set_value(pdomslr_,bonus);  
-  }
-  if (flag_pdomsrl) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw) als->computeDominanceSourcePos(tpdomsrl,_lfw,_rfw,&cost,&bonus,
-                                           &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pdoms_nrl);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(pdomsrl_,cost);
-    estimated_features->set_value(pdomsrl_,bonus); 
-  }
-  if (flag_pgdomslr) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw) als->computeDominanceSourcePos(tpgdomslr,_lfw,_rfw,&cost,&bonus,
-                                           &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pgdoms_nlr,0);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(pgdomslr_,cost);
-    estimated_features->set_value(pgdomslr_,bonus);  
-  }
-  if (flag_pgdomsrl) {    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw) als->computeDominanceSourcePos(tpgdomsrl,_lfw,_rfw,&cost,&bonus,
-                                           &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pgdoms_nrl);
-    if (isFinal&&!explicit_soseos) {
-      cost += bonus;
-      bonus = 0;
-    }
-    features->set_value(pgdomsrl_,cost);
-    estimated_features->set_value(pgdomsrl_,bonus); 
-  }
-
-
-  if (flag_bdoms) {
-    cost=0; bonus=0; bdoms_state_mono=0; bdoms_state_nonmono=0; 
-    if (!nofw)
-      als->computeBorderDominanceSource(tbdoms,&cost,&bonus,
-        &bdoms_state_mono, &bdoms_state_nonmono,*edge.rule_, ant_contexts, sfw);
-    features->set_value(bdoms_,cost);
-    estimated_features->set_value(bdoms_,bonus); 
-  }
-  if (flag_orit) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    if (!nofw) als->computeOrientationTarget(torit,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); 
-    if (DEBUG) cerr << "cost=" << cost << ", bonus=" << bonus << ", bo1=" << bo1 << ", bo1_bonus=" << bo1_bonus << ", bo2=" << bo2 << ", bo2_bonus=" << bo2_bonus << endl;
-    features->set_value(orit_,cost); 
-    //features->set_value(orit_bo1_,bo1); 
-    //features->set_value(orit_bo2_,bo2);
-    estimated_features->set_value(orit_,bonus);
-    //estimated_features->set_value(orit_bo1_,bo1_bonus);
-    //estimated_features->set_value(orit_bo2_,bo2_bonus);
-  }
-  if (flag_orit_backward) {
-    cost=0; bonus=0;
-    if (!nofw) als->computeOrientationTargetBackward(torit,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus);
-    features->set_value(orit_backward_,cost);
-    estimated_features->set_value(orit_backward_,bonus);
-  }
-  if (flag_domt) {
-    cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0;
-    WordID _lfw=-1; int start = edge.i_;
-    WordID _rfw=-1; int end   = edge.j_;
-    if (smeta.HasReference()) {
-      const Lattice l = smeta.GetReference();
-      for (int idx=start-1; idx>=0; idx--) {
-        if (l.size()>0)
-          if (tfw.find(l[idx][0].label) !=tfw.end()) {
-            _lfw=l[idx][0].label; break;
-          }
-      }
-      for (int idx=end; idx<l.size(); idx++) { // end or end+1
-        if (l[idx].size()>0)
-          if (tfw.find(l[idx][0].label)!=tfw.end()) {
-            _rfw=l[idx][0].label; break;
-          }
-      }
-    }
-    //neighboringFWs(smeta.GetReference(),edge.i_,edge.j_,tfw,&_lfw,&_rfw);
-    if (!nofw) als->computeDominanceTarget(tdomt,_lfw,_rfw,&cost,&bonus,
-                                           &bo1,&bo1_bonus,&bo2,&bo2_bonus);
-    features->set_value(domt_,cost); 
-    //features->set_value(domt_bo1_,bo1); 
-    //features->set_value(domt_bo2_,bo2);
-    estimated_features->set_value(domt_,bonus);
-    //estimated_features->set_value(domt_bo1_,bo1_bonus);
-    //estimated_features->set_value(domt_bo2_,bo2_bonus);
-  }
-  int* vcontext = reinterpret_cast<int *>(context);
-  if (!nofw) {
-    als->BorderingSFWsOnly();
-    als->BorderingTFWsOnly();
-    als->simplify(vcontext);  
-  } else {
-    als->simplify_nofw(vcontext);
-  }
-  vcontext[50] = DoubleToInteger(bdoms_state_mono);
-  vcontext[51] = DoubleToInteger(bdoms_state_nonmono);
-  vcontext[STATE_SIZE-1] = Alignment::link(edge.i_,edge.j_); 
-  if (DEBUG) {
-    cerr << "state@traverse = ";
-    for (int idx=0; idx<STATE_SIZE; idx++) cerr << idx << "." << vcontext[idx] << " ";
-    cerr << endl;
-    cerr << "bdoms_state_mono=" << bdoms_state_mono << ", state[50]=" << IntegerToDouble(vcontext[50]) << endl;
-    cerr << "bdoms_state_nonmono=" << bdoms_state_nonmono << ", state[51]=" << IntegerToDouble(vcontext[51]) << endl;
-  }
-}
-
-int Dwarf::DoubleToInteger(double val) {
-  float x = (float)val;
-  float* px = &x;
-  int* pix = reinterpret_cast<int *>(px);
-  return *pix; 
-}
-
-double Dwarf::IntegerToDouble(int val) {
-  int *py = &val;
-  float* pd = reinterpret_cast<float *>(py);
-  return (double)*pd;
-}
-
-void Dwarf::neighboringFWs(const Lattice& l, const int& i, const int& j, const map<WordID,int>& fw_hash, int* lfw, int* rfw) {
-  *lfw=0; *rfw=0;
-  int idx=i-l[i][0].dist2next;
-  while (idx>=0) {
-    if (l[idx].size()>0) { 
-      if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) {
-        lfw++;  
-      }
-    }
-    idx-=l[idx][0].dist2next;
-  }
-  idx=j+l[j][0].dist2next;
-  while (idx<l.size()) {
-    if (l[idx].size()>0) { 
-      if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) {
-        rfw++;
-      }
-    }
-    idx+=l[idx][0].dist2next;
-  }
-}
-
-bool Dwarf::readOrientation(CountTable* table, const std::string& filename, std::map<WordID,int> *fw, bool pos) {
-  // the input format is
-  // source target 0 1 2 3 4 0 1 2 3 4
-  // 0 -> MA, 1 -> RA, 2 -> MG, 3 -> RG, 4 -> NO_NEIGHBOR
-  // first 01234 corresponds to the left neighbor, the second 01234 corresponds to the right neighbor
-  // append 2 more at the end as precomputed total
-  
-  // TONS of hack here. CountTable should be wrapped as a class  
-  // TODO: check whether the file exists or not, return false if not
-  if (DEBUG) cerr << "  readOrientation(" << filename << ", pos=" << pos << ")" << endl;
-  ReadFile rf(filename);  
-  istream& in = *rf.stream();
-  table->setup(24,pos);
-  table->ultimate = new int[24];
-  for (int i=0; i<24; i++) table->ultimate[i]=0;
-  ostringstream oss;
-  while (in) {
-    string line;
-    getline(in,line);
-    if (line=="") break;
-    istringstream tokenizer(line);
-    string sourceidx, source, target, word;
-    tokenizer >> source >> target; 
-    if (pos) {
-      sourceidx = source;
-      source = sourceidx.substr(0,sourceidx.find_last_of("/"));
-    }
-    if (fw->find(TD::Convert(source))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source),1));
-
-
-    int* element = new int[24];
-    element[5] = 0;
-    for (int i=0; i<5; i++) {
-      element[i] = 0;
-      if (tokenizer >> word) element[i] = atoi(word.c_str());
-      element[5] += element[i];
-    }
-    element[11] = 0;
-    for (int i=6; i<11; i++) {
-      element[i] = 0;
-      if (tokenizer >> word) element[i] = atoi(word.c_str());
-      element[11] += element[i];
-    }
-    element[17] = 0;
-    for (int i=12; i<17; i++) {
-      element[i] = 0;
-      if (tokenizer >> word) element[i] = atoi(word.c_str());
-      element[17] += element[i];
-    }
-    element[23] = 0;
-    for (int i=18; i<23; i++) {
-      element[i] = 0;
-      if (tokenizer >> word) element[i] = atoi(word.c_str());
-      element[23] += element[i];
-    }
-    for (int i=0; i<24; i++) table->ultimate[i] += element[i];
-    oss << source << " " << target;
-    WordID key_id = TD::Convert(oss.str());
-    oss.str("");
-    if (table->model.find(key_id)!=table->model.end()) {  
-      for (int i=0; i<24; i++) table->model[key_id][i]+=element[i];
-    } else {
-      int* el2 = new int[24];
-      for (int i=0; i<24; i++) el2[i] = element[i];
-      table->model.insert(pair<WordID,int*>(key_id,el2));
-    }
-    
-    oss << source;
-    key_id = TD::Convert(oss.str());
-    oss.str("");
-    if (table->model.find(key_id)!=table->model.end()) {    
-      for (int i=0; i<24; i++) table->model[key_id][i]+=element[i];
-    } else {
-      int* el2 = new int[24];
-      for (int i=0; i<24; i++) el2[i] = element[i];
-      table->model.insert(pair<WordID,int*>(key_id,el2));
-    }
-
-    if (pos) {
-      oss << sourceidx << " " << target;
-      key_id = TD::Convert(oss.str());
-      oss.str(""); 
-      if (table->model.find(key_id)!=table->model.end()) {
-        for (int i=0; i<24; i++) table->model[key_id][i]+=element[i];
-      } else {
-        int* el2 = new int[24];
-        for (int i=0; i<24; i++) el2[i] = element[i];
-        table->model.insert(pair<WordID,int*>(key_id,el2));
-      }
-    }
-    delete[] element;
-  }  
-  return true;    
-}
-
-bool Dwarf::readList(const std::string& filename, std::map<WordID,int>* fw) {
-  ReadFile rf(filename);
-  istream& in = *rf.stream();
-  while (in) {
-    string word;
-    getline(in,word);
-    if (fw->find(TD::Convert(word))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(word),1)); 
-  }
-  return true;
-}
-
-bool Dwarf::readDominance(CountTable* table, const std::string& filename, std::map<WordID,int>* fw, bool pos) {
-  // the input format is 
-  // source1 source2 target1 target2 0 1 2 3
-  // 0 -> dontcase 1->leftfirst 2->rightfirst 3->neither 
-  if (DEBUG) cerr << "readDominance(" << filename << ",pos="<< pos << ")" << endl;
-  ReadFile rf(filename);
-  istream& in = *rf.stream();
-  table->ultimate = new int[5];
-  table->setup(5,pos);
-  for (int i=0; i<5; i++) table->ultimate[i]=0;
-  while (in) {
-    string line, word;
-    getline(in,line);
-    if (line=="") break;
-    string source1idx, source2idx, target1, target2, source1, source2;
-    ostringstream oss; 
-    WordID key_id;
-    istringstream tokenizer(line);
-    tokenizer >> source1 >> source2 >> target1 >> target2; 
-    if (pos) {
-      source1idx = source1;
-      source2idx = source2;
-      source1 = source1idx.substr(0,source1idx.find_last_of("/"));
-      source2 = source2idx.substr(0,source2idx.find_last_of("/"));
-    }
-    if (fw->find(TD::Convert(source1))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source1),1));
-    if (fw->find(TD::Convert(source2))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source2),1));
-
-    int* element = new int[5];
-    element[4]=0;
-    for (int i=0; i<4; i++) {
-      element[i]  = 0;
-      if (tokenizer >> word) element[i] = atoi(word.c_str());
-      element[4]+=element[i];
-    }
-    for (int i=0; i<5; i++) table->ultimate[i] += element[i];
-
-    oss << source1 << " " << source2 << " " << target1 << " " << target2;
-    key_id = TD::Convert(oss.str());
-    oss.str("");
-    if (table->model.find(key_id)!=table->model.end()) { 
-      for (int i=0; i<5; i++) table->model[key_id][i]+=element[i];
-    } else {
-      int* el2 = new int[5]; 
-      for (int i=0; i<5; i++) el2[i]=element[i];
-      table->model.insert(pair<WordID,int*>(key_id,el2));
-    }
-
-    oss << source1 << " " << source2;
-    key_id = TD::Convert(oss.str());
-    oss.str("");
-    if (table->model.find(key_id)!=table->model.end()) {  
-      for (int i=0; i<5; i++) table->model[key_id][i]+=element[i];
-    } else {
-      int* el2 = new int[5]; 
-      for (int i=0; i<5; i++) el2[i]=element[i];
-      table->model.insert(pair<WordID,int*>(key_id,el2));
-    }
-
-    if (pos) {
-      oss << source1idx << " " << source2idx << " " << target1 << " " << target2;
-      key_id = TD::Convert(oss.str());
-      oss.str("");
-      if (table->model.find(key_id)!=table->model.end()) {  
-        for (int i=0; i<5; i++) table->model[key_id][i]+=element[i];
-      } else {
-        int* el2 = new int[5]; 
-        for (int i=0; i<5; i++) el2[i]=element[i];
-        table->model.insert(pair<WordID,int*>(key_id,el2));
-      }
-    }
-    delete element;
-  }
-
-  return true;    
-}
-
-bool Dwarf::readTags(const std::string& filename, std::map<WordID,WordID>* tags) {
-  ReadFile rf(filename);
-  istream& in = *rf.stream();
-  while(in) {
-    string line, word, tag;
-    getline(in,line);
-    if (line=="") break;
-    istringstream tokenizer(line);
-    tokenizer >> tag >> word;
-    tags->insert(pair<WordID,WordID>(TD::Convert(word),TD::Convert(tag)));
-  }
-  return true;
-}
-
-bool Dwarf::generalizeOrientation(CountTable* table, const std::map<WordID,WordID>& tags, bool pos) {
-  map<string,int*> generalized;
-  for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) {
-    string source, target;
-    istringstream tokenizer(TD::Convert(it->first));
-    tokenizer >> source >> target;
-    string idx = "";
-    if (pos) {
-      int found = source.find_last_of("/");
-      if (found!=string::npos && found>0) { 
-        idx = source.substr(found+1);
-        source = source.substr(0,found);
-      }
-    }
-    map<WordID,WordID>::const_iterator tags_iter = tags.find(TD::Convert(source));
-    if (tags_iter!=tags.end()) {
-      ostringstream genkey;
-      genkey << TD::Convert(tags_iter->second);
-      if (idx!="") genkey << "/" << idx;
-      if (target!="") genkey << " " << target;
-      int* model;
-      if (generalized.find(genkey.str())!=generalized.end()) {
-        model = generalized[genkey.str()];
-        for (int i=0; i<24; i++) model[i] += it->second[i];
-      } else {
-        int* el = new int[24];
-        for (int i=0; i<24; i++) el[i] = it->second[i];
-        generalized.insert(pair<string,int*>(genkey.str(),el));
-      }
-    }
-  }
-  for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) {
-    string source, target;
-    istringstream tokenizer(TD::Convert(it->first));
-    tokenizer >> source >> target;
-    string idx = "";
-    if (pos) {
-      int found = source.find_last_of("/");
-      if (found!=string::npos && found>0) {
-        idx = source.substr(found+1);
-        source = source.substr(0,found);
-      }
-    }
-    map<WordID,WordID>::const_iterator tags_iter = tags.find(TD::Convert(source));
-    if (tags_iter!=tags.end()) {
-      ostringstream genkey;
-      genkey << TD::Convert(tags_iter->second);
-      if (idx!="") genkey << "/" << idx;
-      if (target!="") genkey << " " << target;
-      if (generalized.find(genkey.str())!=generalized.end()) {
-        delete it->second;
-        it->second = generalized[genkey.str()];
-      }
-    }
-  }
-  return false; // no idea if this is right
-}
- 
-
-
-bool Dwarf::generalizeDominance(CountTable* table, const std::map<WordID,WordID>& tags, bool pos) {
-  map<string,int*> generalized;
-  ostringstream oss;
-  for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) {
-    string source1, source2, target1, target2;
-    string idx1 = ""; string idx2 = "";
-    istringstream tokenizer(TD::Convert(it->first));
-    tokenizer >> source1 >> source2 >> target1 >> target2;
-    if (DEBUG) cerr << "source1=|" << source1 << "|, source2=|" << source2 << "|, target1=|" << target1 << "|, target2=|" << target2 << "|" << endl;
-    if (pos) {
-      int found1 = source1.find_last_of("/");
-      int found2 = source2.find_last_of("/");
-      if (found1!=string::npos && found2!=string::npos && found1>0 && found2>0) {
-        idx1 = source1.substr(found1+1);
-        source1 = source1.substr(0,found1);
-        idx2 = source2.substr(found2+1);
-        source2 = source2.substr(0,found2);
-      }
-    }
-    if (DEBUG) 
-      cerr << "[U]source1='" << source1 << "', idx1='"<< idx1 << "', source2='" << source2 << "', idx2='"<< idx2 << "', target1='" << target1 << "', target2='" << target2 << "'" << endl;
-    map<WordID,WordID>::const_iterator tags_iter1 = tags.find(TD::Convert(source1));
-    map<WordID,WordID>::const_iterator tags_iter2 = tags.find(TD::Convert(source2));
-    if (tags_iter1!=tags.end()) 
-      source1 = TD::Convert(tags_iter1->second);
-    oss << source1;
-    if (idx1!="") oss << "/" << idx1;
-    if (tags_iter2!=tags.end())
-      source2 = TD::Convert(tags_iter2->second);
-    oss << " " << source2;
-    if (idx2!="") oss << "/" << idx2;
-    if (target1!="" && target2!="") oss << " " << target1 << " " << target2;
-    
-    if (DEBUG) cerr << "generalized key = '" << oss.str() << "'" << endl; 
-    if (generalized.find(oss.str())!=generalized.end()) {
-      int* model = generalized[oss.str()];
-      for (int i=0; i<5; i++) model[i] += it->second[i];
-    } else {
-      int* model = new int[5];
-      for (int i=0; i<5; i++) model[i] = it->second[i];
-      generalized.insert(pair<string,int*>(oss.str(),model));
-    }    
-    oss.str("");
-  }
-  
-  if (DEBUG) {
-    for (map<string,int*>::const_iterator it=generalized.begin(); it!=generalized.end(); it++) {
-      cerr << "GENERALIZED = " << it->first << ", ";
-      for (int i=0; i<5; i++) cerr << it->second[i] << " ";
-      cerr << endl;
-    }
-  }
-
-  for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) {
-    string source1, source2, target1, target2;
-    string idx1 = ""; string idx2 = "";
-    istringstream tokenizer(TD::Convert(it->first));
-    tokenizer >> source1 >> source2 >> target1 >> target2;
-    if (pos) {
-      int found1 = source1.find_last_of("/");
-      int found2 = source2.find_last_of("/");
-      if (found1!=string::npos && found2!=string::npos && found1>0 && found2>0) {
-        idx1 = source1.substr(found1+1);
-        source1 = source1.substr(0,found1);
-        idx2 = source2.substr(found2+1);
-        source2 = source2.substr(0,found2);
-      }
-    }
-    map<WordID,WordID>::const_iterator tags_iter1 = tags.find(TD::Convert(source1));
-    map<WordID,WordID>::const_iterator tags_iter2 = tags.find(TD::Convert(source2));
-    if (tags_iter1!=tags.end())
-      source1 = TD::Convert(tags_iter1->second);
-    oss << source1;
-    if (idx1!="") oss << "/" << idx1;
-    if (tags_iter2!=tags.end())
-      source2 = TD::Convert(tags_iter2->second);
-    oss << " " << source2;
-    if (idx2!="") oss << "/" << idx2;
-    if (target1!="" && target2!="") oss << " " << target1 << " " << target2;
-    
-    if (generalized.find(oss.str())!=generalized.end()) {
-      if (DEBUG) cerr << " generalizing "<< TD::Convert(it->first) << " into " << oss.str() << endl; 
-      if (DEBUG) {
-        cerr << "  model from ";
-        for (int i=0; i<5; i++) cerr << it->second[i] << " "; 
-        cerr << endl;
-      }
-      delete it->second;
-      it->second = generalized[oss.str()];
-      if (DEBUG) {
-        cerr << "  into ";
-        for (int i=0; i<5; i++) cerr << it->second[i] << " "; 
-        cerr << endl;
-      }
-    }    
-    oss.str("");
-  }
-
-}