diff options
Diffstat (limited to 'decoder/ff_dwarf.cc')
-rw-r--r-- | decoder/ff_dwarf.cc | 894 |
1 files changed, 0 insertions, 894 deletions
diff --git a/decoder/ff_dwarf.cc b/decoder/ff_dwarf.cc deleted file mode 100644 index fe7a472e..00000000 --- a/decoder/ff_dwarf.cc +++ /dev/null @@ -1,894 +0,0 @@ -#include <vector> -#include <sstream> -#include <fstream> -#include <string> -#include <iostream> -#include <map> -#include "hg.h" -#include "ff_dwarf.h" -#include "dwarf.h" -#include "wordid.h" -#include "tdict.h" -#include "filelib.h" -#include "sentence_metadata.h" -#include "stringlib.h" - -using namespace std; - -Dwarf::Dwarf(const std::string& param) { -/* Param is a space separated string which contains any or all of the following: - oris|orit|doms|domt=filename - e.g. oris=/fs/clip-galep3eval/hendra/z2e/oris128.gz -*/ - sSOS="<s>"; - sEOS="</s>"; - kSOS=TD::Convert(sSOS); - kEOS=TD::Convert(sEOS); - kGOAL=TD::Convert("S")*-1; - _sent_id = (int *)malloc(sizeof(int)); - *_sent_id = -1; - if (DEBUG) cerr << "here = " << *_sent_id << endl; - _fwcount = (int *)malloc(sizeof(int)); - *_fwcount = -1; - cerr << "initializing dwarf" << endl; - flag_oris=false; flag_orit=false; flag_doms=false; flag_domt=false; flag_tfw_count=false; - flag_bdoms=false; flag_porislr=false, flag_porisrl=false, flag_goris=false; flag_pgorislr=false, flag_pgorisrl=false; - flag_pdomslr=false; flag_pdomsrl=false; flag_pgdomslr=false; flag_pgdomsrl=false; flag_gdoms=false; - flag_oris_backward=false; flag_orit_backward=false; - explicit_soseos=false; - SetStateSize(STATE_SIZE*sizeof(int)); - als = new Alignment(); - als->clearAls(Alignment::MAX_WORDS,Alignment::MAX_WORDS); - istringstream iss(param); string w; - while(iss >> w) { - int equal = w.find_first_of("="); - if (equal!=string::npos) { - string model = w.substr(0,equal); - vector<string> params; - Tokenize(w.substr(equal+1),',',¶ms); - string fn = params[0]; - if (model == "minfreq") { - cerr << "model minfreq " << fn << endl; - als->setFreqCutoff(atoi(fn.c_str())); - } else if (model == "oris") { - flag_oris = readOrientation(&toris,fn,&sfw); - if (flag_oris) { - oris_ = FD::Convert("OrientationSource"); - //oris_bo1_ = FD::Convert("OrientationSource_BO1"); - //oris_bo2_ = FD::Convert("OrientationSource_BO2"); - } - if (params.size()>1) als->setAlphaOris(atof(params[1].c_str())); - if (params.size()>2) als->setBetaOris(atof(params[2].c_str())); - } else if (model == "porislr") { - flag_porislr = readOrientation(&tporislr,fn,&sfw,true); - poris_nlr = 0; - if (flag_porislr) { - porislr_ = FD::Convert("OrientationSourcePositionfulLeftRight"); - } - if (params.size()>1) poris_nlr = atoi(params[1].c_str()); - if (DEBUG) cerr << " maximum poris depth=" << poris_nlr << endl; - } else if (model == "porisrl") { - flag_porisrl = readOrientation(&tporisrl,fn,&sfw,true); - poris_nrl = 0; - if (flag_porisrl) { - porisrl_ = FD::Convert("OrientationSourcePositionfulRightLeft"); - } - if (params.size()>1) poris_nrl = atoi(params[1].c_str()); - if (DEBUG) cerr << " maximum poris depth=" << poris_nrl << endl; - } else if (model=="goris") { - flag_goris = readOrientation(&tgoris,fn,&sfw); - if (flag_goris) { - goris_ = FD::Convert("OrientationSourceGeneralized"); - } - if (params.size()>1) { - readTags(params[1],&tags); - generalizeOrientation(&tgoris,tags); - } - } else if (model=="pgorislr") { - flag_pgorislr = readOrientation(&tpgorislr,fn,&sfw,true); - pgoris_nlr = 0; - if (flag_pgorislr) { - pgorislr_ = FD::Convert("OrientationSourceGeneralizedPositionfulLeftRight"); - } - if (DEBUG) { - cerr << "BEFORE GENERALIZATION" << endl; - tpgorislr.print(); - } - if (params.size()>1) pgoris_nlr = atoi(params[1].c_str()); - if (params.size()>2) { - readTags(params[2],&tags); - generalizeOrientation(&tpgorislr,tags,true); - } - if (DEBUG) { - cerr << "AFTER GENERALIZATION" << endl; - tpgorislr.print(); - } - } else if (model=="pgorisrl") { - flag_pgorisrl = readOrientation(&tpgorisrl,fn,&sfw,true); - pgoris_nrl = 0; - if (flag_pgorisrl) { - pgorisrl_ = FD::Convert("OrientationSourceGeneralizedPositionfulLeftRight"); - } - if (params.size()>1) pgoris_nrl = atoi(params[1].c_str()); - if (params.size()>2) { - readTags(params[2],&tags); - generalizeOrientation(&tpgorisrl,tags,true); - } - } else if (model == "oris_backward") { - flag_oris_backward = true; - if (!flag_oris) readOrientation(&toris,fn,&sfw); - oris_backward_ = FD::Convert("OrientationSourceBackward"); - if (params.size()>1) als->setAlphaOris(atof(params[1].c_str())); - if (params.size()>2) als->setBetaOris(atof(params[2].c_str())); - } else if (model == "orit") { - flag_orit = readOrientation(&torit,fn,&tfw); - if (flag_orit) { - orit_ = FD::Convert("OrientationTarget"); - //orit_bo1_ = FD::Convert("OrientationTarget_BO1"); - //orit_bo2_ = FD::Convert("OrientationTarget_BO2"); - } - if (params.size()>1) als->setAlphaOrit(atof(params[1].c_str())); - if (params.size()>2) als->setBetaOrit(atof(params[2].c_str())); - } else if (model == "orit_backward") { - flag_orit_backward = true; - if (!flag_orit) readOrientation(&torit,fn,&tfw); - orit_backward_ = FD::Convert("OrientationTargetBackward"); - if (params.size()>1) als->setAlphaOrit(atof(params[1].c_str())); - if (params.size()>2) als->setBetaOrit(atof(params[2].c_str())); - } else if (model == "doms") { - flag_doms = readDominance(&tdoms,fn,&sfw); - if (flag_doms) { - doms_ = FD::Convert("DominanceSource"); - //doms_bo1_ = FD::Convert("DominanceSource_BO1"); - //doms_bo2_ = FD::Convert("DominanceSource_BO2"); - } - if (params.size()>1) als->setAlphaDoms(atof(params[1].c_str())); - if (params.size()>2) als->setBetaDoms(atof(params[2].c_str())); - } else if (model == "pdomsrl") { - flag_pdomsrl = readDominance(&tpdomsrl,fn,&sfw,true); - if (flag_pdomsrl) { - pdomsrl_ = FD::Convert("DominanceSourcePositionfulRightLeft"); - } - if (params.size()>1) pdoms_nrl = atoi(params[1].c_str()); - } else if (model == "pdomslr") { - flag_pdomslr = readDominance(&tpdomslr,fn,&sfw,true); - tpdomslr.print(); - if (flag_pdomslr) { - pdomslr_ = FD::Convert("DominanceSourcePositionfulLeftRight"); - } - if (params.size()>1) pdoms_nlr = atoi(params[1].c_str()); - } else if (model == "pgdomsrl") { - flag_pgdomsrl = readDominance(&tpgdomsrl,fn,&sfw,true); - if (flag_pgdomsrl) { - pgdomsrl_ = FD::Convert("DominanceSourceGeneralizedPositionfulRightLeft"); - } - if (params.size()>1) pgdoms_nrl = atoi(params[1].c_str()); - if (params.size()>2) { - readTags(params[2],&tags); - generalizeDominance(&tpgdomsrl,tags,true); - } - } else if (model == "pgdomslr") { - flag_pgdomslr = readDominance(&tpgdomslr,fn,&sfw,true); - if (flag_pgdomslr) { - pgdomslr_ = FD::Convert("DominanceSourceGeneralizedPositionfulLeftRight"); - } - if (params.size()>1) pgdoms_nlr = atoi(params[1].c_str()); - if (params.size()>2) { - readTags(params[2],&tags); - if (DEBUG) { - for (map<WordID,WordID>::const_iterator it=tags.begin(); it!=tags.end(); it++) { - cerr << "tags = " << TD::Convert(it->first) << ", " << TD::Convert(it->second) << endl; - } - } - generalizeDominance(&tpgdomslr,tags,true); - } - if (DEBUG) tpgdomslr.print(); - } else if (model == "bdoms") { - flag_bdoms = readDominance(&tbdoms,fn,&sfw); - if (flag_bdoms) { - bdoms_ = FD::Convert("BorderDominanceSource"); - } - } else if (model == "domt") { - flag_domt = readDominance(&tdomt,fn,&tfw); - if (flag_domt) { - domt_ = FD::Convert("DominanceTarget"); - //domt_bo1_ = FD::Convert("DominanceTarget_BO1"); - //domt_bo2_ = FD::Convert("DominanceTarget_BO2"); - } - if (params.size()>1) als->setAlphaDomt(atof(params[1].c_str())); - if (params.size()>2) als->setBetaDomt(atof(params[2].c_str())); - } else if (model== "tfw_count") { - flag_tfw_count = readList(fn,&tfw); - tfw_count_ = FD::Convert("TargetFunctionWordsCount"); - } else { - cerr << "DWARF doesn't understand this model: " << model << endl; - } - } else { - if (w=="tfw_count") { - flag_tfw_count = true; - tfw_count_ = FD::Convert("TargetFunctionWordsCount"); - } else if (w=="oris_backward") { - flag_oris_backward = true; - oris_backward_ = FD::Convert("OrientationSourceBackward"); - } else if (w=="orit_backward") { - flag_orit_backward = true; - orit_backward_ = FD::Convert("OrientationTargetBackward"); - } else if (w=="explicit_soseos") { - explicit_soseos=true; - } else { - cerr << "DWARF doesn't need this param: " << param << endl; - } - } - } - for (map<WordID,int>::const_iterator it=sfw.begin(); it!=sfw.end() && DEBUG; it++) { - cerr << " FW:" << TD::Convert(it->first) << endl; - } -} - -void Dwarf::TraversalFeaturesImpl(const SentenceMetadata& smeta, - const Hypergraph::Edge& edge, - const std::vector<const void*>& ant_contexts, - SparseVector<double>* features, - SparseVector<double>* estimated_features, - void* context) const { - if (DEBUG) cerr << "TraversalFeaturesImpl" << endl; - double cost, bonus, bo1, bo2, bo1_bonus, bo2_bonus; - double bdoms_state_mono= 0; double bdoms_state_nonmono = 0; - TRule r = *edge.rule_; - if (DEBUG) cerr << " sent_id=" << *_sent_id << ", " << smeta.GetSentenceID() << endl; - if (DEBUG) cerr << "rule = " << r.AsString() << endl; - if (DEBUG) cerr << "rule[i,j] = " << edge.i_ << "," << edge.j_ << endl; - if (*_sent_id != smeta.GetSentenceID()) { //new sentence - *_sent_id = smeta.GetSentenceID(); - const Lattice l = smeta.GetSourceLattice(); - *_fwcount=0; - for (int i=0; i<smeta.GetSourceLength(); i++) { - if (sfw.find(l[i][0].label)!=sfw.end()) { - *_fwcount+=1; - } - } - if (DEBUG) cerr << "new sentence[" << *_sent_id << "]="<<*_fwcount<<endl; - } - bool nofw = als->prepare(*edge.rule_, ant_contexts, sfw, tfw,smeta.GetSourceLattice(),edge.i_,edge.j_); - bool isFinal = (edge.i_==0 && edge.j_==smeta.GetSourceLength() && r.GetLHS()==kGOAL); - // prepare *nofw* outputs whether the resulting alignment, contains function words or not - // if not, the models do not have to be calcualted and *simplify* is very simple - if (DEBUG) cerr << "nofw = " << nofw << endl; - if (flag_tfw_count) { - double count = 0; - for (int i=0; i<r.e_.size(); i++) { - if (tfw.find(r.e_[i])!=tfw.end()) count++; - } - features->set_value(tfw_count_,count); - } - if (flag_oris) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) als->computeOrientationSource(toris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(oris_,cost); - //features->set_value(oris_bo1_,bo1); - //features->set_value(oris_bo2_,bo2); - estimated_features->set_value(oris_,bonus); - //estimated_features->set_value(oris_bo1_,bo1_bonus); - //estimated_features->set_value(oris_bo2_,bo2_bonus); - } - if (flag_porislr) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) - als->computeOrientationSourcePos(tporislr,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,poris_nlr,0); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(porislr_,cost); - estimated_features->set_value(porislr_,bonus); - } - if (flag_porisrl) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) - als->computeOrientationSourcePos(tporisrl,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,poris_nrl); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(porisrl_,cost); - estimated_features->set_value(porisrl_,bonus); - } - if (flag_pgorislr) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) - als->computeOrientationSourcePos(tpgorislr,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pgoris_nlr,0); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(pgorislr_,cost); - estimated_features->set_value(pgorislr_,bonus); - } - if (flag_pgorisrl) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) - als->computeOrientationSourcePos(tpgorisrl,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pgoris_nrl); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(pgorisrl_,cost); - estimated_features->set_value(pgorisrl_,bonus); - } - if (flag_goris) { - cost=0; bonus=0; - if (!nofw) als->computeOrientationSource(tgoris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(goris_,cost); - estimated_features->set_value(goris_,bonus); - } - if (flag_oris_backward) { - cost=0; bonus=0; - if (!nofw) - als->computeOrientationSourceBackward(toris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(oris_backward_,cost); - estimated_features->set_value(oris_backward_,bonus); - } - WordID _lfw = kSOS; - WordID _rfw = kEOS; - if (flag_doms || flag_pdomslr || flag_pdomsrl || flag_pgdomslr || flag_pgdomsrl) { - if (DEBUG) cerr << " seeking lfw and rfw" << endl; - int start = edge.i_; - int end = edge.j_; - if (DEBUG) cerr << " start=" << start << ", end=" << end << endl; - const Lattice l = smeta.GetSourceLattice(); - for (int idx=start-1; idx>=0; idx--) { - if (DEBUG) cerr << " checking idx=" << idx << ", label=" << l[idx][0].label << "-" << TD::Convert(l[idx][0].label) << endl; - if (sfw.find(l[idx][0].label) !=sfw.end()) { - if (DEBUG) cerr << "+"; - _lfw=l[idx][0].label; break; - } - } - for (int idx=end; idx<l.size(); idx++) { // end or end+1 - if (DEBUG) cerr << " checking idx=" << idx << ", label=" << l[idx][0].label << "-" << TD::Convert(l[idx][0].label) << endl; - if (sfw.find(l[idx][0].label)!=sfw.end()) { - if (DEBUG) cerr << "."; - _rfw=l[idx][0].label; break; - } - } - if (isFinal&&!explicit_soseos) { - _lfw=kSOS; _rfw=kEOS; - } - } - if (flag_doms) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) als->computeDominanceSource(tdoms,_lfw,_rfw,&cost,&bonus, - &bo1,&bo1_bonus,&bo2,&bo2_bonus); - if (DEBUG) cerr << " COST=" << cost << ", BONUS=" << bonus << endl; - if (isFinal&&!explicit_soseos) { - cost += bonus; - if (DEBUG) cerr << " final and !explicit_soseos, thus cost = " << cost << endl; - bonus = 0; - } - features->set_value(doms_,cost); - estimated_features->set_value(doms_,bonus); - } - if (flag_pdomslr) { - if (DEBUG) cerr << " flag_pdomslr true, nofw=" << nofw << endl; - if (DEBUG) cerr << " lfw=" << _lfw << ", rfw=" << _rfw << endl; - if (DEBUG) cerr << " kSOS=" << kSOS << ", kEOS=" << kEOS << endl; - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) als->computeDominanceSourcePos(tpdomslr,_lfw,_rfw,&cost,&bonus, - &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pdoms_nlr,0); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(pdomslr_,cost); - estimated_features->set_value(pdomslr_,bonus); - } - if (flag_pdomsrl) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) als->computeDominanceSourcePos(tpdomsrl,_lfw,_rfw,&cost,&bonus, - &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pdoms_nrl); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(pdomsrl_,cost); - estimated_features->set_value(pdomsrl_,bonus); - } - if (flag_pgdomslr) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) als->computeDominanceSourcePos(tpgdomslr,_lfw,_rfw,&cost,&bonus, - &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pgdoms_nlr,0); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(pgdomslr_,cost); - estimated_features->set_value(pgdomslr_,bonus); - } - if (flag_pgdomsrl) { cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) als->computeDominanceSourcePos(tpgdomsrl,_lfw,_rfw,&cost,&bonus, - &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pgdoms_nrl); - if (isFinal&&!explicit_soseos) { - cost += bonus; - bonus = 0; - } - features->set_value(pgdomsrl_,cost); - estimated_features->set_value(pgdomsrl_,bonus); - } - - - if (flag_bdoms) { - cost=0; bonus=0; bdoms_state_mono=0; bdoms_state_nonmono=0; - if (!nofw) - als->computeBorderDominanceSource(tbdoms,&cost,&bonus, - &bdoms_state_mono, &bdoms_state_nonmono,*edge.rule_, ant_contexts, sfw); - features->set_value(bdoms_,cost); - estimated_features->set_value(bdoms_,bonus); - } - if (flag_orit) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - if (!nofw) als->computeOrientationTarget(torit,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); - if (DEBUG) cerr << "cost=" << cost << ", bonus=" << bonus << ", bo1=" << bo1 << ", bo1_bonus=" << bo1_bonus << ", bo2=" << bo2 << ", bo2_bonus=" << bo2_bonus << endl; - features->set_value(orit_,cost); - //features->set_value(orit_bo1_,bo1); - //features->set_value(orit_bo2_,bo2); - estimated_features->set_value(orit_,bonus); - //estimated_features->set_value(orit_bo1_,bo1_bonus); - //estimated_features->set_value(orit_bo2_,bo2_bonus); - } - if (flag_orit_backward) { - cost=0; bonus=0; - if (!nofw) als->computeOrientationTargetBackward(torit,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); - features->set_value(orit_backward_,cost); - estimated_features->set_value(orit_backward_,bonus); - } - if (flag_domt) { - cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; - WordID _lfw=-1; int start = edge.i_; - WordID _rfw=-1; int end = edge.j_; - if (smeta.HasReference()) { - const Lattice l = smeta.GetReference(); - for (int idx=start-1; idx>=0; idx--) { - if (l.size()>0) - if (tfw.find(l[idx][0].label) !=tfw.end()) { - _lfw=l[idx][0].label; break; - } - } - for (int idx=end; idx<l.size(); idx++) { // end or end+1 - if (l[idx].size()>0) - if (tfw.find(l[idx][0].label)!=tfw.end()) { - _rfw=l[idx][0].label; break; - } - } - } - //neighboringFWs(smeta.GetReference(),edge.i_,edge.j_,tfw,&_lfw,&_rfw); - if (!nofw) als->computeDominanceTarget(tdomt,_lfw,_rfw,&cost,&bonus, - &bo1,&bo1_bonus,&bo2,&bo2_bonus); - features->set_value(domt_,cost); - //features->set_value(domt_bo1_,bo1); - //features->set_value(domt_bo2_,bo2); - estimated_features->set_value(domt_,bonus); - //estimated_features->set_value(domt_bo1_,bo1_bonus); - //estimated_features->set_value(domt_bo2_,bo2_bonus); - } - int* vcontext = reinterpret_cast<int *>(context); - if (!nofw) { - als->BorderingSFWsOnly(); - als->BorderingTFWsOnly(); - als->simplify(vcontext); - } else { - als->simplify_nofw(vcontext); - } - vcontext[50] = DoubleToInteger(bdoms_state_mono); - vcontext[51] = DoubleToInteger(bdoms_state_nonmono); - vcontext[STATE_SIZE-1] = Alignment::link(edge.i_,edge.j_); - if (DEBUG) { - cerr << "state@traverse = "; - for (int idx=0; idx<STATE_SIZE; idx++) cerr << idx << "." << vcontext[idx] << " "; - cerr << endl; - cerr << "bdoms_state_mono=" << bdoms_state_mono << ", state[50]=" << IntegerToDouble(vcontext[50]) << endl; - cerr << "bdoms_state_nonmono=" << bdoms_state_nonmono << ", state[51]=" << IntegerToDouble(vcontext[51]) << endl; - } -} - -int Dwarf::DoubleToInteger(double val) { - float x = (float)val; - float* px = &x; - int* pix = reinterpret_cast<int *>(px); - return *pix; -} - -double Dwarf::IntegerToDouble(int val) { - int *py = &val; - float* pd = reinterpret_cast<float *>(py); - return (double)*pd; -} - -void Dwarf::neighboringFWs(const Lattice& l, const int& i, const int& j, const map<WordID,int>& fw_hash, int* lfw, int* rfw) { - *lfw=0; *rfw=0; - int idx=i-l[i][0].dist2next; - while (idx>=0) { - if (l[idx].size()>0) { - if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) { - lfw++; - } - } - idx-=l[idx][0].dist2next; - } - idx=j+l[j][0].dist2next; - while (idx<l.size()) { - if (l[idx].size()>0) { - if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) { - rfw++; - } - } - idx+=l[idx][0].dist2next; - } -} - -bool Dwarf::readOrientation(CountTable* table, const std::string& filename, std::map<WordID,int> *fw, bool pos) { - // the input format is - // source target 0 1 2 3 4 0 1 2 3 4 - // 0 -> MA, 1 -> RA, 2 -> MG, 3 -> RG, 4 -> NO_NEIGHBOR - // first 01234 corresponds to the left neighbor, the second 01234 corresponds to the right neighbor - // append 2 more at the end as precomputed total - - // TONS of hack here. CountTable should be wrapped as a class - // TODO: check whether the file exists or not, return false if not - if (DEBUG) cerr << " readOrientation(" << filename << ", pos=" << pos << ")" << endl; - ReadFile rf(filename); - istream& in = *rf.stream(); - table->setup(24,pos); - table->ultimate = new int[24]; - for (int i=0; i<24; i++) table->ultimate[i]=0; - ostringstream oss; - while (in) { - string line; - getline(in,line); - if (line=="") break; - istringstream tokenizer(line); - string sourceidx, source, target, word; - tokenizer >> source >> target; - if (pos) { - sourceidx = source; - source = sourceidx.substr(0,sourceidx.find_last_of("/")); - } - if (fw->find(TD::Convert(source))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source),1)); - - - int* element = new int[24]; - element[5] = 0; - for (int i=0; i<5; i++) { - element[i] = 0; - if (tokenizer >> word) element[i] = atoi(word.c_str()); - element[5] += element[i]; - } - element[11] = 0; - for (int i=6; i<11; i++) { - element[i] = 0; - if (tokenizer >> word) element[i] = atoi(word.c_str()); - element[11] += element[i]; - } - element[17] = 0; - for (int i=12; i<17; i++) { - element[i] = 0; - if (tokenizer >> word) element[i] = atoi(word.c_str()); - element[17] += element[i]; - } - element[23] = 0; - for (int i=18; i<23; i++) { - element[i] = 0; - if (tokenizer >> word) element[i] = atoi(word.c_str()); - element[23] += element[i]; - } - for (int i=0; i<24; i++) table->ultimate[i] += element[i]; - oss << source << " " << target; - WordID key_id = TD::Convert(oss.str()); - oss.str(""); - if (table->model.find(key_id)!=table->model.end()) { - for (int i=0; i<24; i++) table->model[key_id][i]+=element[i]; - } else { - int* el2 = new int[24]; - for (int i=0; i<24; i++) el2[i] = element[i]; - table->model.insert(pair<WordID,int*>(key_id,el2)); - } - - oss << source; - key_id = TD::Convert(oss.str()); - oss.str(""); - if (table->model.find(key_id)!=table->model.end()) { - for (int i=0; i<24; i++) table->model[key_id][i]+=element[i]; - } else { - int* el2 = new int[24]; - for (int i=0; i<24; i++) el2[i] = element[i]; - table->model.insert(pair<WordID,int*>(key_id,el2)); - } - - if (pos) { - oss << sourceidx << " " << target; - key_id = TD::Convert(oss.str()); - oss.str(""); - if (table->model.find(key_id)!=table->model.end()) { - for (int i=0; i<24; i++) table->model[key_id][i]+=element[i]; - } else { - int* el2 = new int[24]; - for (int i=0; i<24; i++) el2[i] = element[i]; - table->model.insert(pair<WordID,int*>(key_id,el2)); - } - } - delete[] element; - } - return true; -} - -bool Dwarf::readList(const std::string& filename, std::map<WordID,int>* fw) { - ReadFile rf(filename); - istream& in = *rf.stream(); - while (in) { - string word; - getline(in,word); - if (fw->find(TD::Convert(word))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(word),1)); - } - return true; -} - -bool Dwarf::readDominance(CountTable* table, const std::string& filename, std::map<WordID,int>* fw, bool pos) { - // the input format is - // source1 source2 target1 target2 0 1 2 3 - // 0 -> dontcase 1->leftfirst 2->rightfirst 3->neither - if (DEBUG) cerr << "readDominance(" << filename << ",pos="<< pos << ")" << endl; - ReadFile rf(filename); - istream& in = *rf.stream(); - table->ultimate = new int[5]; - table->setup(5,pos); - for (int i=0; i<5; i++) table->ultimate[i]=0; - while (in) { - string line, word; - getline(in,line); - if (line=="") break; - string source1idx, source2idx, target1, target2, source1, source2; - ostringstream oss; - WordID key_id; - istringstream tokenizer(line); - tokenizer >> source1 >> source2 >> target1 >> target2; - if (pos) { - source1idx = source1; - source2idx = source2; - source1 = source1idx.substr(0,source1idx.find_last_of("/")); - source2 = source2idx.substr(0,source2idx.find_last_of("/")); - } - if (fw->find(TD::Convert(source1))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source1),1)); - if (fw->find(TD::Convert(source2))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source2),1)); - - int* element = new int[5]; - element[4]=0; - for (int i=0; i<4; i++) { - element[i] = 0; - if (tokenizer >> word) element[i] = atoi(word.c_str()); - element[4]+=element[i]; - } - for (int i=0; i<5; i++) table->ultimate[i] += element[i]; - - oss << source1 << " " << source2 << " " << target1 << " " << target2; - key_id = TD::Convert(oss.str()); - oss.str(""); - if (table->model.find(key_id)!=table->model.end()) { - for (int i=0; i<5; i++) table->model[key_id][i]+=element[i]; - } else { - int* el2 = new int[5]; - for (int i=0; i<5; i++) el2[i]=element[i]; - table->model.insert(pair<WordID,int*>(key_id,el2)); - } - - oss << source1 << " " << source2; - key_id = TD::Convert(oss.str()); - oss.str(""); - if (table->model.find(key_id)!=table->model.end()) { - for (int i=0; i<5; i++) table->model[key_id][i]+=element[i]; - } else { - int* el2 = new int[5]; - for (int i=0; i<5; i++) el2[i]=element[i]; - table->model.insert(pair<WordID,int*>(key_id,el2)); - } - - if (pos) { - oss << source1idx << " " << source2idx << " " << target1 << " " << target2; - key_id = TD::Convert(oss.str()); - oss.str(""); - if (table->model.find(key_id)!=table->model.end()) { - for (int i=0; i<5; i++) table->model[key_id][i]+=element[i]; - } else { - int* el2 = new int[5]; - for (int i=0; i<5; i++) el2[i]=element[i]; - table->model.insert(pair<WordID,int*>(key_id,el2)); - } - } - delete element; - } - - return true; -} - -bool Dwarf::readTags(const std::string& filename, std::map<WordID,WordID>* tags) { - ReadFile rf(filename); - istream& in = *rf.stream(); - while(in) { - string line, word, tag; - getline(in,line); - if (line=="") break; - istringstream tokenizer(line); - tokenizer >> tag >> word; - tags->insert(pair<WordID,WordID>(TD::Convert(word),TD::Convert(tag))); - } - return true; -} - -bool Dwarf::generalizeOrientation(CountTable* table, const std::map<WordID,WordID>& tags, bool pos) { - map<string,int*> generalized; - for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) { - string source, target; - istringstream tokenizer(TD::Convert(it->first)); - tokenizer >> source >> target; - string idx = ""; - if (pos) { - int found = source.find_last_of("/"); - if (found!=string::npos && found>0) { - idx = source.substr(found+1); - source = source.substr(0,found); - } - } - map<WordID,WordID>::const_iterator tags_iter = tags.find(TD::Convert(source)); - if (tags_iter!=tags.end()) { - ostringstream genkey; - genkey << TD::Convert(tags_iter->second); - if (idx!="") genkey << "/" << idx; - if (target!="") genkey << " " << target; - int* model; - if (generalized.find(genkey.str())!=generalized.end()) { - model = generalized[genkey.str()]; - for (int i=0; i<24; i++) model[i] += it->second[i]; - } else { - int* el = new int[24]; - for (int i=0; i<24; i++) el[i] = it->second[i]; - generalized.insert(pair<string,int*>(genkey.str(),el)); - } - } - } - for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) { - string source, target; - istringstream tokenizer(TD::Convert(it->first)); - tokenizer >> source >> target; - string idx = ""; - if (pos) { - int found = source.find_last_of("/"); - if (found!=string::npos && found>0) { - idx = source.substr(found+1); - source = source.substr(0,found); - } - } - map<WordID,WordID>::const_iterator tags_iter = tags.find(TD::Convert(source)); - if (tags_iter!=tags.end()) { - ostringstream genkey; - genkey << TD::Convert(tags_iter->second); - if (idx!="") genkey << "/" << idx; - if (target!="") genkey << " " << target; - if (generalized.find(genkey.str())!=generalized.end()) { - delete it->second; - it->second = generalized[genkey.str()]; - } - } - } - return false; // no idea if this is right -} - - - -bool Dwarf::generalizeDominance(CountTable* table, const std::map<WordID,WordID>& tags, bool pos) { - map<string,int*> generalized; - ostringstream oss; - for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) { - string source1, source2, target1, target2; - string idx1 = ""; string idx2 = ""; - istringstream tokenizer(TD::Convert(it->first)); - tokenizer >> source1 >> source2 >> target1 >> target2; - if (DEBUG) cerr << "source1=|" << source1 << "|, source2=|" << source2 << "|, target1=|" << target1 << "|, target2=|" << target2 << "|" << endl; - if (pos) { - int found1 = source1.find_last_of("/"); - int found2 = source2.find_last_of("/"); - if (found1!=string::npos && found2!=string::npos && found1>0 && found2>0) { - idx1 = source1.substr(found1+1); - source1 = source1.substr(0,found1); - idx2 = source2.substr(found2+1); - source2 = source2.substr(0,found2); - } - } - if (DEBUG) - cerr << "[U]source1='" << source1 << "', idx1='"<< idx1 << "', source2='" << source2 << "', idx2='"<< idx2 << "', target1='" << target1 << "', target2='" << target2 << "'" << endl; - map<WordID,WordID>::const_iterator tags_iter1 = tags.find(TD::Convert(source1)); - map<WordID,WordID>::const_iterator tags_iter2 = tags.find(TD::Convert(source2)); - if (tags_iter1!=tags.end()) - source1 = TD::Convert(tags_iter1->second); - oss << source1; - if (idx1!="") oss << "/" << idx1; - if (tags_iter2!=tags.end()) - source2 = TD::Convert(tags_iter2->second); - oss << " " << source2; - if (idx2!="") oss << "/" << idx2; - if (target1!="" && target2!="") oss << " " << target1 << " " << target2; - - if (DEBUG) cerr << "generalized key = '" << oss.str() << "'" << endl; - if (generalized.find(oss.str())!=generalized.end()) { - int* model = generalized[oss.str()]; - for (int i=0; i<5; i++) model[i] += it->second[i]; - } else { - int* model = new int[5]; - for (int i=0; i<5; i++) model[i] = it->second[i]; - generalized.insert(pair<string,int*>(oss.str(),model)); - } - oss.str(""); - } - - if (DEBUG) { - for (map<string,int*>::const_iterator it=generalized.begin(); it!=generalized.end(); it++) { - cerr << "GENERALIZED = " << it->first << ", "; - for (int i=0; i<5; i++) cerr << it->second[i] << " "; - cerr << endl; - } - } - - for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) { - string source1, source2, target1, target2; - string idx1 = ""; string idx2 = ""; - istringstream tokenizer(TD::Convert(it->first)); - tokenizer >> source1 >> source2 >> target1 >> target2; - if (pos) { - int found1 = source1.find_last_of("/"); - int found2 = source2.find_last_of("/"); - if (found1!=string::npos && found2!=string::npos && found1>0 && found2>0) { - idx1 = source1.substr(found1+1); - source1 = source1.substr(0,found1); - idx2 = source2.substr(found2+1); - source2 = source2.substr(0,found2); - } - } - map<WordID,WordID>::const_iterator tags_iter1 = tags.find(TD::Convert(source1)); - map<WordID,WordID>::const_iterator tags_iter2 = tags.find(TD::Convert(source2)); - if (tags_iter1!=tags.end()) - source1 = TD::Convert(tags_iter1->second); - oss << source1; - if (idx1!="") oss << "/" << idx1; - if (tags_iter2!=tags.end()) - source2 = TD::Convert(tags_iter2->second); - oss << " " << source2; - if (idx2!="") oss << "/" << idx2; - if (target1!="" && target2!="") oss << " " << target1 << " " << target2; - - if (generalized.find(oss.str())!=generalized.end()) { - if (DEBUG) cerr << " generalizing "<< TD::Convert(it->first) << " into " << oss.str() << endl; - if (DEBUG) { - cerr << " model from "; - for (int i=0; i<5; i++) cerr << it->second[i] << " "; - cerr << endl; - } - delete it->second; - it->second = generalized[oss.str()]; - if (DEBUG) { - cerr << " into "; - for (int i=0; i<5; i++) cerr << it->second[i] << " "; - cerr << endl; - } - } - oss.str(""); - } - -} |