diff options
Diffstat (limited to 'decoder/ff_dwarf.cc')
| -rw-r--r-- | decoder/ff_dwarf.cc | 893 |
1 files changed, 893 insertions, 0 deletions
diff --git a/decoder/ff_dwarf.cc b/decoder/ff_dwarf.cc new file mode 100644 index 00000000..3daa85ac --- /dev/null +++ b/decoder/ff_dwarf.cc @@ -0,0 +1,893 @@ +#include <vector> +#include <sstream> +#include <fstream> +#include <string> +#include <iostream> +#include <map> +#include "ff_dwarf.h" +#include "dwarf.h" +#include "wordid.h" +#include "tdict.h" +#include "filelib.h" +#include "sentence_metadata.h" +#include "stringlib.h" + +using namespace std; + +Dwarf::Dwarf(const std::string& param) { +/* Param is a space separated string which contains any or all of the following: + oris|orit|doms|domt=filename + e.g. oris=/fs/clip-galep3eval/hendra/z2e/oris128.gz +*/ + sSOS="<s>"; + sEOS="</s>"; + kSOS=TD::Convert(sSOS); + kEOS=TD::Convert(sEOS); + kGOAL=TD::Convert("S")*-1; + _sent_id = (int *)malloc(sizeof(int)); + *_sent_id = -1; + if (DEBUG) cerr << "here = " << *_sent_id << endl; + _fwcount = (int *)malloc(sizeof(int)); + *_fwcount = -1; + cerr << "initializing dwarf" << endl; + flag_oris=false; flag_orit=false; flag_doms=false; flag_domt=false; flag_tfw_count=false; + flag_bdoms=false; flag_porislr=false, flag_porisrl=false, flag_goris=false; flag_pgorislr=false, flag_pgorisrl=false; + flag_pdomslr=false; flag_pdomsrl=false; flag_pgdomslr=false; flag_pgdomsrl=false; flag_gdoms=false; + flag_oris_backward=false; flag_orit_backward=false; + explicit_soseos=false; + SetStateSize(STATE_SIZE*sizeof(int)); + als = new Alignment(); + als->clearAls(Alignment::MAX_WORDS,Alignment::MAX_WORDS); + istringstream iss(param); string w; + while(iss >> w) { + int equal = w.find_first_of("="); + if (equal!=string::npos) { + string model = w.substr(0,equal); + vector<string> params; + Tokenize(w.substr(equal+1),',',¶ms); + string fn = params[0]; + if (model == "minfreq") { + cerr << "model minfreq " << fn << endl; + als->setFreqCutoff(atoi(fn.c_str())); + } else if (model == "oris") { + flag_oris = readOrientation(&toris,fn,&sfw); + if (flag_oris) { + oris_ = FD::Convert("OrientationSource"); + //oris_bo1_ = FD::Convert("OrientationSource_BO1"); + //oris_bo2_ = FD::Convert("OrientationSource_BO2"); + } + if (params.size()>1) als->setAlphaOris(atof(params[1].c_str())); + if (params.size()>2) als->setBetaOris(atof(params[2].c_str())); + } else if (model == "porislr") { + flag_porislr = readOrientation(&tporislr,fn,&sfw,true); + poris_nlr = 0; + if (flag_porislr) { + porislr_ = FD::Convert("OrientationSourcePositionfulLeftRight"); + } + if (params.size()>1) poris_nlr = atoi(params[1].c_str()); + if (DEBUG) cerr << " maximum poris depth=" << poris_nlr << endl; + } else if (model == "porisrl") { + flag_porisrl = readOrientation(&tporisrl,fn,&sfw,true); + poris_nrl = 0; + if (flag_porisrl) { + porisrl_ = FD::Convert("OrientationSourcePositionfulRightLeft"); + } + if (params.size()>1) poris_nrl = atoi(params[1].c_str()); + if (DEBUG) cerr << " maximum poris depth=" << poris_nrl << endl; + } else if (model=="goris") { + flag_goris = readOrientation(&tgoris,fn,&sfw); + if (flag_goris) { + goris_ = FD::Convert("OrientationSourceGeneralized"); + } + if (params.size()>1) { + readTags(params[1],&tags); + generalizeOrientation(&tgoris,tags); + } + } else if (model=="pgorislr") { + flag_pgorislr = readOrientation(&tpgorislr,fn,&sfw,true); + pgoris_nlr = 0; + if (flag_pgorislr) { + pgorislr_ = FD::Convert("OrientationSourceGeneralizedPositionfulLeftRight"); + } + if (DEBUG) { + cerr << "BEFORE GENERALIZATION" << endl; + tpgorislr.print(); + } + if (params.size()>1) pgoris_nlr = atoi(params[1].c_str()); + if (params.size()>2) { + readTags(params[2],&tags); + generalizeOrientation(&tpgorislr,tags,true); + } + if (DEBUG) { + cerr << "AFTER GENERALIZATION" << endl; + tpgorislr.print(); + } + } else if (model=="pgorisrl") { + flag_pgorisrl = readOrientation(&tpgorisrl,fn,&sfw,true); + pgoris_nrl = 0; + if (flag_pgorisrl) { + pgorisrl_ = FD::Convert("OrientationSourceGeneralizedPositionfulLeftRight"); + } + if (params.size()>1) pgoris_nrl = atoi(params[1].c_str()); + if (params.size()>2) { + readTags(params[2],&tags); + generalizeOrientation(&tpgorisrl,tags,true); + } + } else if (model == "oris_backward") { + flag_oris_backward = true; + if (!flag_oris) readOrientation(&toris,fn,&sfw); + oris_backward_ = FD::Convert("OrientationSourceBackward"); + if (params.size()>1) als->setAlphaOris(atof(params[1].c_str())); + if (params.size()>2) als->setBetaOris(atof(params[2].c_str())); + } else if (model == "orit") { + flag_orit = readOrientation(&torit,fn,&tfw); + if (flag_orit) { + orit_ = FD::Convert("OrientationTarget"); + //orit_bo1_ = FD::Convert("OrientationTarget_BO1"); + //orit_bo2_ = FD::Convert("OrientationTarget_BO2"); + } + if (params.size()>1) als->setAlphaOrit(atof(params[1].c_str())); + if (params.size()>2) als->setBetaOrit(atof(params[2].c_str())); + } else if (model == "orit_backward") { + flag_orit_backward = true; + if (!flag_orit) readOrientation(&torit,fn,&tfw); + orit_backward_ = FD::Convert("OrientationTargetBackward"); + if (params.size()>1) als->setAlphaOrit(atof(params[1].c_str())); + if (params.size()>2) als->setBetaOrit(atof(params[2].c_str())); + } else if (model == "doms") { + flag_doms = readDominance(&tdoms,fn,&sfw); + if (flag_doms) { + doms_ = FD::Convert("DominanceSource"); + //doms_bo1_ = FD::Convert("DominanceSource_BO1"); + //doms_bo2_ = FD::Convert("DominanceSource_BO2"); + } + if (params.size()>1) als->setAlphaDoms(atof(params[1].c_str())); + if (params.size()>2) als->setBetaDoms(atof(params[2].c_str())); + } else if (model == "pdomsrl") { + flag_pdomsrl = readDominance(&tpdomsrl,fn,&sfw,true); + if (flag_pdomsrl) { + pdomsrl_ = FD::Convert("DominanceSourcePositionfulRightLeft"); + } + if (params.size()>1) pdoms_nrl = atoi(params[1].c_str()); + } else if (model == "pdomslr") { + flag_pdomslr = readDominance(&tpdomslr,fn,&sfw,true); + tpdomslr.print(); + if (flag_pdomslr) { + pdomslr_ = FD::Convert("DominanceSourcePositionfulLeftRight"); + } + if (params.size()>1) pdoms_nlr = atoi(params[1].c_str()); + } else if (model == "pgdomsrl") { + flag_pgdomsrl = readDominance(&tpgdomsrl,fn,&sfw,true); + if (flag_pgdomsrl) { + pgdomsrl_ = FD::Convert("DominanceSourceGeneralizedPositionfulRightLeft"); + } + if (params.size()>1) pgdoms_nrl = atoi(params[1].c_str()); + if (params.size()>2) { + readTags(params[2],&tags); + generalizeDominance(&tpgdomsrl,tags,true); + } + } else if (model == "pgdomslr") { + flag_pgdomslr = readDominance(&tpgdomslr,fn,&sfw,true); + if (flag_pgdomslr) { + pgdomslr_ = FD::Convert("DominanceSourceGeneralizedPositionfulLeftRight"); + } + if (params.size()>1) pgdoms_nlr = atoi(params[1].c_str()); + if (params.size()>2) { + readTags(params[2],&tags); + if (DEBUG) { + for (map<WordID,WordID>::const_iterator it=tags.begin(); it!=tags.end(); it++) { + cerr << "tags = " << TD::Convert(it->first) << ", " << TD::Convert(it->second) << endl; + } + } + generalizeDominance(&tpgdomslr,tags,true); + } + if (DEBUG) tpgdomslr.print(); + } else if (model == "bdoms") { + flag_bdoms = readDominance(&tbdoms,fn,&sfw); + if (flag_bdoms) { + bdoms_ = FD::Convert("BorderDominanceSource"); + } + } else if (model == "domt") { + flag_domt = readDominance(&tdomt,fn,&tfw); + if (flag_domt) { + domt_ = FD::Convert("DominanceTarget"); + //domt_bo1_ = FD::Convert("DominanceTarget_BO1"); + //domt_bo2_ = FD::Convert("DominanceTarget_BO2"); + } + if (params.size()>1) als->setAlphaDomt(atof(params[1].c_str())); + if (params.size()>2) als->setBetaDomt(atof(params[2].c_str())); + } else if (model== "tfw_count") { + flag_tfw_count = readList(fn,&tfw); + tfw_count_ = FD::Convert("TargetFunctionWordsCount"); + } else { + cerr << "DWARF doesn't understand this model: " << model << endl; + } + } else { + if (w=="tfw_count") { + flag_tfw_count = true; + tfw_count_ = FD::Convert("TargetFunctionWordsCount"); + } else if (w=="oris_backward") { + flag_oris_backward = true; + oris_backward_ = FD::Convert("OrientationSourceBackward"); + } else if (w=="orit_backward") { + flag_orit_backward = true; + orit_backward_ = FD::Convert("OrientationTargetBackward"); + } else if (w=="explicit_soseos") { + explicit_soseos=true; + } else { + cerr << "DWARF doesn't need this param: " << param << endl; + } + } + } + for (map<WordID,int>::const_iterator it=sfw.begin(); it!=sfw.end() && DEBUG; it++) { + cerr << " FW:" << TD::Convert(it->first) << endl; + } +} + +void Dwarf::TraversalFeaturesImpl(const SentenceMetadata& smeta, + const Hypergraph::Edge& edge, + const std::vector<const void*>& ant_contexts, + SparseVector<double>* features, + SparseVector<double>* estimated_features, + void* context) const { + if (DEBUG) cerr << "TraversalFeaturesImpl" << endl; + double cost, bonus, bo1, bo2, bo1_bonus, bo2_bonus; + double bdoms_state_mono= 0; double bdoms_state_nonmono = 0; + TRule r = *edge.rule_; + if (DEBUG) cerr << " sent_id=" << *_sent_id << ", " << smeta.GetSentenceID() << endl; + if (DEBUG) cerr << "rule = " << r.AsString() << endl; + if (DEBUG) cerr << "rule[i,j] = " << edge.i_ << "," << edge.j_ << endl; + if (*_sent_id != smeta.GetSentenceID()) { //new sentence + *_sent_id = smeta.GetSentenceID(); + const Lattice l = smeta.GetSourceLattice(); + *_fwcount=0; + for (int i=0; i<smeta.GetSourceLength(); i++) { + if (sfw.find(l[i][0].label)!=sfw.end()) { + *_fwcount+=1; + } + } + if (DEBUG) cerr << "new sentence[" << *_sent_id << "]="<<*_fwcount<<endl; + } + bool nofw = als->prepare(*edge.rule_, ant_contexts, sfw, tfw,smeta.GetSourceLattice(),edge.i_,edge.j_); + bool isFinal = (edge.i_==0 && edge.j_==smeta.GetSourceLength() && r.GetLHS()==kGOAL); + // prepare *nofw* outputs whether the resulting alignment, contains function words or not + // if not, the models do not have to be calcualted and *simplify* is very simple + if (DEBUG) cerr << "nofw = " << nofw << endl; + if (flag_tfw_count) { + double count = 0; + for (int i=0; i<r.e_.size(); i++) { + if (tfw.find(r.e_[i])!=tfw.end()) count++; + } + features->set_value(tfw_count_,count); + } + if (flag_oris) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) als->computeOrientationSource(toris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(oris_,cost); + //features->set_value(oris_bo1_,bo1); + //features->set_value(oris_bo2_,bo2); + estimated_features->set_value(oris_,bonus); + //estimated_features->set_value(oris_bo1_,bo1_bonus); + //estimated_features->set_value(oris_bo2_,bo2_bonus); + } + if (flag_porislr) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) + als->computeOrientationSourcePos(tporislr,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,poris_nlr,0); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(porislr_,cost); + estimated_features->set_value(porislr_,bonus); + } + if (flag_porisrl) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) + als->computeOrientationSourcePos(tporisrl,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,poris_nrl); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(porisrl_,cost); + estimated_features->set_value(porisrl_,bonus); + } + if (flag_pgorislr) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) + als->computeOrientationSourcePos(tpgorislr,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pgoris_nlr,0); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(pgorislr_,cost); + estimated_features->set_value(pgorislr_,bonus); + } + if (flag_pgorisrl) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) + als->computeOrientationSourcePos(tpgorisrl,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pgoris_nrl); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(pgorisrl_,cost); + estimated_features->set_value(pgorisrl_,bonus); + } + if (flag_goris) { + cost=0; bonus=0; + if (!nofw) als->computeOrientationSource(tgoris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(goris_,cost); + estimated_features->set_value(goris_,bonus); + } + if (flag_oris_backward) { + cost=0; bonus=0; + if (!nofw) + als->computeOrientationSourceBackward(toris,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(oris_backward_,cost); + estimated_features->set_value(oris_backward_,bonus); + } + WordID _lfw = kSOS; + WordID _rfw = kEOS; + if (flag_doms || flag_pdomslr || flag_pdomsrl || flag_pgdomslr || flag_pgdomsrl) { + if (DEBUG) cerr << " seeking lfw and rfw" << endl; + int start = edge.i_; + int end = edge.j_; + if (DEBUG) cerr << " start=" << start << ", end=" << end << endl; + const Lattice l = smeta.GetSourceLattice(); + for (int idx=start-1; idx>=0; idx--) { + if (DEBUG) cerr << " checking idx=" << idx << ", label=" << l[idx][0].label << "-" << TD::Convert(l[idx][0].label) << endl; + if (sfw.find(l[idx][0].label) !=sfw.end()) { + if (DEBUG) cerr << "+"; + _lfw=l[idx][0].label; break; + } + } + for (int idx=end; idx<l.size(); idx++) { // end or end+1 + if (DEBUG) cerr << " checking idx=" << idx << ", label=" << l[idx][0].label << "-" << TD::Convert(l[idx][0].label) << endl; + if (sfw.find(l[idx][0].label)!=sfw.end()) { + if (DEBUG) cerr << "."; + _rfw=l[idx][0].label; break; + } + } + if (isFinal&&!explicit_soseos) { + _lfw=kSOS; _rfw=kEOS; + } + } + if (flag_doms) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) als->computeDominanceSource(tdoms,_lfw,_rfw,&cost,&bonus, + &bo1,&bo1_bonus,&bo2,&bo2_bonus); + if (DEBUG) cerr << " COST=" << cost << ", BONUS=" << bonus << endl; + if (isFinal&&!explicit_soseos) { + cost += bonus; + if (DEBUG) cerr << " final and !explicit_soseos, thus cost = " << cost << endl; + bonus = 0; + } + features->set_value(doms_,cost); + estimated_features->set_value(doms_,bonus); + } + if (flag_pdomslr) { + if (DEBUG) cerr << " flag_pdomslr true, nofw=" << nofw << endl; + if (DEBUG) cerr << " lfw=" << _lfw << ", rfw=" << _rfw << endl; + if (DEBUG) cerr << " kSOS=" << kSOS << ", kEOS=" << kEOS << endl; + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) als->computeDominanceSourcePos(tpdomslr,_lfw,_rfw,&cost,&bonus, + &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pdoms_nlr,0); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(pdomslr_,cost); + estimated_features->set_value(pdomslr_,bonus); + } + if (flag_pdomsrl) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) als->computeDominanceSourcePos(tpdomsrl,_lfw,_rfw,&cost,&bonus, + &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pdoms_nrl); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(pdomsrl_,cost); + estimated_features->set_value(pdomsrl_,bonus); + } + if (flag_pgdomslr) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) als->computeDominanceSourcePos(tpgdomslr,_lfw,_rfw,&cost,&bonus, + &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,pgdoms_nlr,0); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(pgdomslr_,cost); + estimated_features->set_value(pgdomslr_,bonus); + } + if (flag_pgdomsrl) { cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) als->computeDominanceSourcePos(tpgdomsrl,_lfw,_rfw,&cost,&bonus, + &bo1,&bo1_bonus,&bo2,&bo2_bonus,*_fwcount,0,pgdoms_nrl); + if (isFinal&&!explicit_soseos) { + cost += bonus; + bonus = 0; + } + features->set_value(pgdomsrl_,cost); + estimated_features->set_value(pgdomsrl_,bonus); + } + + + if (flag_bdoms) { + cost=0; bonus=0; bdoms_state_mono=0; bdoms_state_nonmono=0; + if (!nofw) + als->computeBorderDominanceSource(tbdoms,&cost,&bonus, + &bdoms_state_mono, &bdoms_state_nonmono,*edge.rule_, ant_contexts, sfw); + features->set_value(bdoms_,cost); + estimated_features->set_value(bdoms_,bonus); + } + if (flag_orit) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + if (!nofw) als->computeOrientationTarget(torit,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); + if (DEBUG) cerr << "cost=" << cost << ", bonus=" << bonus << ", bo1=" << bo1 << ", bo1_bonus=" << bo1_bonus << ", bo2=" << bo2 << ", bo2_bonus=" << bo2_bonus << endl; + features->set_value(orit_,cost); + //features->set_value(orit_bo1_,bo1); + //features->set_value(orit_bo2_,bo2); + estimated_features->set_value(orit_,bonus); + //estimated_features->set_value(orit_bo1_,bo1_bonus); + //estimated_features->set_value(orit_bo2_,bo2_bonus); + } + if (flag_orit_backward) { + cost=0; bonus=0; + if (!nofw) als->computeOrientationTargetBackward(torit,&cost,&bonus,&bo1,&bo1_bonus,&bo2,&bo2_bonus); + features->set_value(orit_backward_,cost); + estimated_features->set_value(orit_backward_,bonus); + } + if (flag_domt) { + cost=0; bonus=0; bo1=0; bo2=0; bo1_bonus=0; bo2_bonus=0; + WordID _lfw=-1; int start = edge.i_; + WordID _rfw=-1; int end = edge.j_; + if (smeta.HasReference()) { + const Lattice l = smeta.GetReference(); + for (int idx=start-1; idx>=0; idx--) { + if (l.size()>0) + if (tfw.find(l[idx][0].label) !=tfw.end()) { + _lfw=l[idx][0].label; break; + } + } + for (int idx=end; idx<l.size(); idx++) { // end or end+1 + if (l[idx].size()>0) + if (tfw.find(l[idx][0].label)!=tfw.end()) { + _rfw=l[idx][0].label; break; + } + } + } + //neighboringFWs(smeta.GetReference(),edge.i_,edge.j_,tfw,&_lfw,&_rfw); + if (!nofw) als->computeDominanceTarget(tdomt,_lfw,_rfw,&cost,&bonus, + &bo1,&bo1_bonus,&bo2,&bo2_bonus); + features->set_value(domt_,cost); + //features->set_value(domt_bo1_,bo1); + //features->set_value(domt_bo2_,bo2); + estimated_features->set_value(domt_,bonus); + //estimated_features->set_value(domt_bo1_,bo1_bonus); + //estimated_features->set_value(domt_bo2_,bo2_bonus); + } + int* vcontext = reinterpret_cast<int *>(context); + if (!nofw) { + als->BorderingSFWsOnly(); + als->BorderingTFWsOnly(); + als->simplify(vcontext); + } else { + als->simplify_nofw(vcontext); + } + vcontext[50] = DoubleToInteger(bdoms_state_mono); + vcontext[51] = DoubleToInteger(bdoms_state_nonmono); + vcontext[STATE_SIZE-1] = Alignment::link(edge.i_,edge.j_); + if (DEBUG) { + cerr << "state@traverse = "; + for (int idx=0; idx<STATE_SIZE; idx++) cerr << idx << "." << vcontext[idx] << " "; + cerr << endl; + cerr << "bdoms_state_mono=" << bdoms_state_mono << ", state[50]=" << IntegerToDouble(vcontext[50]) << endl; + cerr << "bdoms_state_nonmono=" << bdoms_state_nonmono << ", state[51]=" << IntegerToDouble(vcontext[51]) << endl; + } +} + +int Dwarf::DoubleToInteger(double val) { + float x = (float)val; + float* px = &x; + int* pix = reinterpret_cast<int *>(px); + return *pix; +} + +double Dwarf::IntegerToDouble(int val) { + int *py = &val; + float* pd = reinterpret_cast<float *>(py); + return (double)*pd; +} + +void Dwarf::neighboringFWs(const Lattice& l, const int& i, const int& j, const map<WordID,int>& fw_hash, int* lfw, int* rfw) { + *lfw=0; *rfw=0; + int idx=i-l[i][0].dist2next; + while (idx>=0) { + if (l[idx].size()>0) { + if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) { + *lfw++; + } + } + idx-=l[idx][0].dist2next; + } + idx=j+l[j][0].dist2next; + while (idx<l.size()) { + if (l[idx].size()>0) { + if (fw_hash.find(l[idx][0].label)!=fw_hash.end()) { + *rfw++; + } + } + idx+=l[idx][0].dist2next; + } +} + +bool Dwarf::readOrientation(CountTable* table, const std::string& filename, std::map<WordID,int> *fw, bool pos) { + // the input format is + // source target 0 1 2 3 4 0 1 2 3 4 + // 0 -> MA, 1 -> RA, 2 -> MG, 3 -> RG, 4 -> NO_NEIGHBOR + // first 01234 corresponds to the left neighbor, the second 01234 corresponds to the right neighbor + // append 2 more at the end as precomputed total + + // TONS of hack here. CountTable should be wrapped as a class + // TODO: check whether the file exists or not, return false if not + if (DEBUG) cerr << " readOrientation(" << filename << ", pos=" << pos << ")" << endl; + ReadFile rf(filename); + istream& in = *rf.stream(); + table->setup(24,pos); + table->ultimate = new int[24]; + for (int i=0; i<24; i++) table->ultimate[i]=0; + ostringstream oss; + while (in) { + string line; + getline(in,line); + if (line=="") break; + istringstream tokenizer(line); + string sourceidx, source, target, word; + tokenizer >> source >> target; + if (pos) { + sourceidx = source; + source = sourceidx.substr(0,sourceidx.find_last_of("/")); + } + if (fw->find(TD::Convert(source))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source),1)); + + + int* element = new int[24]; + element[5] = 0; + for (int i=0; i<5; i++) { + element[i] = 0; + if (tokenizer >> word) element[i] = atoi(word.c_str()); + element[5] += element[i]; + } + element[11] = 0; + for (int i=6; i<11; i++) { + element[i] = 0; + if (tokenizer >> word) element[i] = atoi(word.c_str()); + element[11] += element[i]; + } + element[17] = 0; + for (int i=12; i<17; i++) { + element[i] = 0; + if (tokenizer >> word) element[i] = atoi(word.c_str()); + element[17] += element[i]; + } + element[23] = 0; + for (int i=18; i<23; i++) { + element[i] = 0; + if (tokenizer >> word) element[i] = atoi(word.c_str()); + element[23] += element[i]; + } + for (int i=0; i<24; i++) table->ultimate[i] += element[i]; + oss << source << " " << target; + WordID key_id = TD::Convert(oss.str()); + oss.str(""); + if (table->model.find(key_id)!=table->model.end()) { + for (int i=0; i<24; i++) table->model[key_id][i]+=element[i]; + } else { + int* el2 = new int[24]; + for (int i=0; i<24; i++) el2[i] = element[i]; + table->model.insert(pair<WordID,int*>(key_id,el2)); + } + + oss << source; + key_id = TD::Convert(oss.str()); + oss.str(""); + if (table->model.find(key_id)!=table->model.end()) { + for (int i=0; i<24; i++) table->model[key_id][i]+=element[i]; + } else { + int* el2 = new int[24]; + for (int i=0; i<24; i++) el2[i] = element[i]; + table->model.insert(pair<WordID,int*>(key_id,el2)); + } + + if (pos) { + oss << sourceidx << " " << target; + key_id = TD::Convert(oss.str()); + oss.str(""); + if (table->model.find(key_id)!=table->model.end()) { + for (int i=0; i<24; i++) table->model[key_id][i]+=element[i]; + } else { + int* el2 = new int[24]; + for (int i=0; i<24; i++) el2[i] = element[i]; + table->model.insert(pair<WordID,int*>(key_id,el2)); + } + } + delete[] element; + } + return true; +} + +bool Dwarf::readList(const std::string& filename, std::map<WordID,int>* fw) { + ReadFile rf(filename); + istream& in = *rf.stream(); + while (in) { + string word; + getline(in,word); + if (fw->find(TD::Convert(word))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(word),1)); + } + return true; +} + +bool Dwarf::readDominance(CountTable* table, const std::string& filename, std::map<WordID,int>* fw, bool pos) { + // the input format is + // source1 source2 target1 target2 0 1 2 3 + // 0 -> dontcase 1->leftfirst 2->rightfirst 3->neither + if (DEBUG) cerr << "readDominance(" << filename << ",pos="<< pos << ")" << endl; + ReadFile rf(filename); + istream& in = *rf.stream(); + table->ultimate = new int[5]; + table->setup(5,pos); + for (int i=0; i<5; i++) table->ultimate[i]=0; + while (in) { + string line, word; + getline(in,line); + if (line=="") break; + string source1idx, source2idx, target1, target2, source1, source2; + ostringstream oss; + WordID key_id; + istringstream tokenizer(line); + tokenizer >> source1 >> source2 >> target1 >> target2; + if (pos) { + source1idx = source1; + source2idx = source2; + source1 = source1idx.substr(0,source1idx.find_last_of("/")); + source2 = source2idx.substr(0,source2idx.find_last_of("/")); + } + if (fw->find(TD::Convert(source1))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source1),1)); + if (fw->find(TD::Convert(source2))==fw->end()) fw->insert(pair<WordID,int>(TD::Convert(source2),1)); + + int* element = new int[5]; + element[4]=0; + for (int i=0; i<4; i++) { + element[i] = 0; + if (tokenizer >> word) element[i] = atoi(word.c_str()); + element[4]+=element[i]; + } + for (int i=0; i<5; i++) table->ultimate[i] += element[i]; + + oss << source1 << " " << source2 << " " << target1 << " " << target2; + key_id = TD::Convert(oss.str()); + oss.str(""); + if (table->model.find(key_id)!=table->model.end()) { + for (int i=0; i<5; i++) table->model[key_id][i]+=element[i]; + } else { + int* el2 = new int[5]; + for (int i=0; i<5; i++) el2[i]=element[i]; + table->model.insert(pair<WordID,int*>(key_id,el2)); + } + + oss << source1 << " " << source2; + key_id = TD::Convert(oss.str()); + oss.str(""); + if (table->model.find(key_id)!=table->model.end()) { + for (int i=0; i<5; i++) table->model[key_id][i]+=element[i]; + } else { + int* el2 = new int[5]; + for (int i=0; i<5; i++) el2[i]=element[i]; + table->model.insert(pair<WordID,int*>(key_id,el2)); + } + + if (pos) { + oss << source1idx << " " << source2idx << " " << target1 << " " << target2; + key_id = TD::Convert(oss.str()); + oss.str(""); + if (table->model.find(key_id)!=table->model.end()) { + for (int i=0; i<5; i++) table->model[key_id][i]+=element[i]; + } else { + int* el2 = new int[5]; + for (int i=0; i<5; i++) el2[i]=element[i]; + table->model.insert(pair<WordID,int*>(key_id,el2)); + } + } + delete element; + } + + return true; +} + +bool Dwarf::readTags(const std::string& filename, std::map<WordID,WordID>* tags) { + ReadFile rf(filename); + istream& in = *rf.stream(); + while(in) { + string line, word, tag; + getline(in,line); + if (line=="") break; + istringstream tokenizer(line); + tokenizer >> tag >> word; + tags->insert(pair<WordID,WordID>(TD::Convert(word),TD::Convert(tag))); + } + return true; +} + +bool Dwarf::generalizeOrientation(CountTable* table, const std::map<WordID,WordID>& tags, bool pos) { + map<string,int*> generalized; + for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) { + string source, target; + istringstream tokenizer(TD::Convert(it->first)); + tokenizer >> source >> target; + string idx = ""; + if (pos) { + int found = source.find_last_of("/"); + if (found!=string::npos && found>0) { + idx = source.substr(found+1); + source = source.substr(0,found); + } + } + map<WordID,WordID>::const_iterator tags_iter = tags.find(TD::Convert(source)); + if (tags_iter!=tags.end()) { + ostringstream genkey; + genkey << TD::Convert(tags_iter->second); + if (idx!="") genkey << "/" << idx; + if (target!="") genkey << " " << target; + int* model; + if (generalized.find(genkey.str())!=generalized.end()) { + model = generalized[genkey.str()]; + for (int i=0; i<24; i++) model[i] += it->second[i]; + } else { + int* el = new int[24]; + for (int i=0; i<24; i++) el[i] = it->second[i]; + generalized.insert(pair<string,int*>(genkey.str(),el)); + } + } + } + for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) { + string source, target; + istringstream tokenizer(TD::Convert(it->first)); + tokenizer >> source >> target; + string idx = ""; + if (pos) { + int found = source.find_last_of("/"); + if (found!=string::npos && found>0) { + idx = source.substr(found+1); + source = source.substr(0,found); + } + } + map<WordID,WordID>::const_iterator tags_iter = tags.find(TD::Convert(source)); + if (tags_iter!=tags.end()) { + ostringstream genkey; + genkey << TD::Convert(tags_iter->second); + if (idx!="") genkey << "/" << idx; + if (target!="") genkey << " " << target; + if (generalized.find(genkey.str())!=generalized.end()) { + delete it->second; + it->second = generalized[genkey.str()]; + } + } + } + +} + + + +bool Dwarf::generalizeDominance(CountTable* table, const std::map<WordID,WordID>& tags, bool pos) { + map<string,int*> generalized; + ostringstream oss; + for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) { + string source1, source2, target1, target2; + string idx1 = ""; string idx2 = ""; + istringstream tokenizer(TD::Convert(it->first)); + tokenizer >> source1 >> source2 >> target1 >> target2; + if (DEBUG) cerr << "source1=|" << source1 << "|, source2=|" << source2 << "|, target1=|" << target1 << "|, target2=|" << target2 << "|" << endl; + if (pos) { + int found1 = source1.find_last_of("/"); + int found2 = source2.find_last_of("/"); + if (found1!=string::npos && found2!=string::npos && found1>0 && found2>0) { + idx1 = source1.substr(found1+1); + source1 = source1.substr(0,found1); + idx2 = source2.substr(found2+1); + source2 = source2.substr(0,found2); + } + } + if (DEBUG) + cerr << "[U]source1='" << source1 << "', idx1='"<< idx1 << "', source2='" << source2 << "', idx2='"<< idx2 << "', target1='" << target1 << "', target2='" << target2 << "'" << endl; + map<WordID,WordID>::const_iterator tags_iter1 = tags.find(TD::Convert(source1)); + map<WordID,WordID>::const_iterator tags_iter2 = tags.find(TD::Convert(source2)); + if (tags_iter1!=tags.end()) + source1 = TD::Convert(tags_iter1->second); + oss << source1; + if (idx1!="") oss << "/" << idx1; + if (tags_iter2!=tags.end()) + source2 = TD::Convert(tags_iter2->second); + oss << " " << source2; + if (idx2!="") oss << "/" << idx2; + if (target1!="" && target2!="") oss << " " << target1 << " " << target2; + + if (DEBUG) cerr << "generalized key = '" << oss.str() << "'" << endl; + if (generalized.find(oss.str())!=generalized.end()) { + int* model = generalized[oss.str()]; + for (int i=0; i<5; i++) model[i] += it->second[i]; + } else { + int* model = new int[5]; + for (int i=0; i<5; i++) model[i] = it->second[i]; + generalized.insert(pair<string,int*>(oss.str(),model)); + } + oss.str(""); + } + + if (DEBUG) { + for (map<string,int*>::const_iterator it=generalized.begin(); it!=generalized.end(); it++) { + cerr << "GENERALIZED = " << it->first << ", "; + for (int i=0; i<5; i++) cerr << it->second[i] << " "; + cerr << endl; + } + } + + for (map<WordID,int*>::iterator it=table->model.begin(); it!=table->model.end(); it++) { + string source1, source2, target1, target2; + string idx1 = ""; string idx2 = ""; + istringstream tokenizer(TD::Convert(it->first)); + tokenizer >> source1 >> source2 >> target1 >> target2; + if (pos) { + int found1 = source1.find_last_of("/"); + int found2 = source2.find_last_of("/"); + if (found1!=string::npos && found2!=string::npos && found1>0 && found2>0) { + idx1 = source1.substr(found1+1); + source1 = source1.substr(0,found1); + idx2 = source2.substr(found2+1); + source2 = source2.substr(0,found2); + } + } + map<WordID,WordID>::const_iterator tags_iter1 = tags.find(TD::Convert(source1)); + map<WordID,WordID>::const_iterator tags_iter2 = tags.find(TD::Convert(source2)); + if (tags_iter1!=tags.end()) + source1 = TD::Convert(tags_iter1->second); + oss << source1; + if (idx1!="") oss << "/" << idx1; + if (tags_iter2!=tags.end()) + source2 = TD::Convert(tags_iter2->second); + oss << " " << source2; + if (idx2!="") oss << "/" << idx2; + if (target1!="" && target2!="") oss << " " << target1 << " " << target2; + + if (generalized.find(oss.str())!=generalized.end()) { + if (DEBUG) cerr << " generalizing "<< TD::Convert(it->first) << " into " << oss.str() << endl; + if (DEBUG) { + cerr << " model from "; + for (int i=0; i<5; i++) cerr << it->second[i] << " "; + cerr << endl; + } + delete it->second; + it->second = generalized[oss.str()]; + if (DEBUG) { + cerr << " into "; + for (int i=0; i<5; i++) cerr << it->second[i] << " "; + cerr << endl; + } + } + oss.str(""); + } + +} |