From 64f43ca5010758b58326d727e359b5908de4fcb0 Mon Sep 17 00:00:00 2001 From: graehl Date: Wed, 28 Jul 2010 07:10:09 +0000 Subject: debugging print - still no idea on .05% difference scoring 3gram using phrases git-svn-id: https://ws10smt.googlecode.com/svn/trunk@446 ec762483-ff6d-05da-a07a-a48fb63a330f --- decoder/cdec.cc | 2 +- decoder/fdict.cc | 1 - decoder/ff_from_fsa.h | 68 +++++++++++++++++++++++++++++++-------------------- decoder/ff_lm.cc | 8 +++--- decoder/ff_lm_fsa.h | 21 ++++++++++------ decoder/hg.h | 4 ++- 6 files changed, 65 insertions(+), 39 deletions(-) (limited to 'decoder') diff --git a/decoder/cdec.cc b/decoder/cdec.cc index 460e9f15..876dee18 100644 --- a/decoder/cdec.cc +++ b/decoder/cdec.cc @@ -133,7 +133,7 @@ void InitCommandLine(int argc, char** argv, OracleBleu &ob, po::variables_map* c ("ctf_no_exhaustive", "Do not fall back to exhaustive parse if coarse-to-fine parsing fails") ("beam_prune", po::value(), "Prune paths from +LM forest, keep paths within exp(alpha>=0)") ("scale_prune_srclen", "scale beams by the input length (in # of tokens; may not be what you want for lattices") - ("promise_power",po::value()->default_value(0), "Give more beam budget to more promising previous-pass nodes when pruning - but allocate the same average beams. 0 means off, 1 means beam proportional to inside*outside prob, n means nth power (affects just --cubepruning_pop_limit)") + ("promise_power",po::value()->default_value(0), "Give more beam budget to more promising previous-pass nodes when pruning - but allocate the same average beams. 0 means off, 1 means beam proportional to inside*outside prob, n means nth power (affects just --cubepruning_pop_limit). note: for the same poplimit, this gives more search error unless very close to 0 (recommend disabled; even 0.01 is slightly worse than 0) which is a bad sign and suggests this isn't doing a good job; further it's slightly slower to LM cube rescore with 0.01 compared to 0, as well as giving (very insignificantly) lower BLEU. TODO: test under more conditions, or try idea with different formula, or prob. cube beams.") ("lexalign_use_null", "Support source-side null words in lexical translation") ("tagger_tagset,t", po::value(), "(Tagger) file containing tag set") ("csplit_output_plf", "(Compound splitter) Output lattice in PLF format") diff --git a/decoder/fdict.cc b/decoder/fdict.cc index da80c260..65187685 100644 --- a/decoder/fdict.cc +++ b/decoder/fdict.cc @@ -111,7 +111,6 @@ std::string UrlDecodeString(const std::string & encoded) { } std::string UrlEncodeString(const std::string & decoded) { - const char * sz_decoded = decoded.c_str(); size_t needed_length = decoded.length() * 3 + 3; char stackalloc[64]; char * buf = needed_length > sizeof(stackalloc)/sizeof(*stackalloc) ? diff --git a/decoder/ff_from_fsa.h b/decoder/ff_from_fsa.h index c517ec64..10ccfe6d 100755 --- a/decoder/ff_from_fsa.h +++ b/decoder/ff_from_fsa.h @@ -3,7 +3,9 @@ #include "ff_fsa.h" -#define FSA_FF_DEBUG 0 +#ifndef FSA_FF_DEBUG +# define FSA_FF_DEBUG 0 +#endif #if FSA_FF_DEBUG # define FSAFFDBG(e,x) FSADBGif(debug(),e,x) # define FSAFFDBGnl(e) FSADBGif_nl(debug(),e) @@ -49,35 +51,36 @@ public: void* out_state) const { TRule const& rule=*edge.rule_; - Sentence const& e = rule.e(); + Sentence const& e = rule.e(); // items in target side of rule typename Impl::Accum accum,h_accum; if (!ssz) { // special case for no state - but still build up longer phrases to score in case FSA overrides ScanPhraseAccum if (Impl::simple_phrase_score) { - // save the effort of building up the contiguous rule phrases - for (int j=0,je=e.size();j=1) // token ff.ScanAccum(smeta,edge,(WordID)e[j],NULL,NULL,&accum); - FSAFFDBG(edge," "<=1) + for (int j=0,ee=e.size();;++j) { // items in target side of rule + for(;;++j) { + if (j>=ee) goto rhs_done; // j may go 1 past ee due to k possibly getting to end + if (RHS_WORD(j)) break; } + // word @j + int k=j; + while(k fsa(ff,smeta,edge); /* fsa holds our current state once we've seen our first M rule or child left-context words. that state scores up the rest of the words at the time, and is replaced by the right state of any full child. at the end, if we've got at least M left words in all, it becomes our right state (otherwise, we don't bother storing the partial state, which might seem useful any time we're built on by a rule that has our variable in the initial position - but without also storing the heuristic for that case, we just end up rescanning from scratch anyway to produce the heuristic. so we just store all 0 bytes if we have less than M left words at the end. */ - for (int j = 0; j < e.size(); ++j) { // items in target side of rule - if (e[j] < 1) { // variable + for (int j = 0,ee=e.size(); j < ee; ++j) { // items in target side of rule + s_rhs_next: + if (!RHS_WORD(j)) { // variable // variables a* are referring to this child derivation state. SP a = ant_contexts[-e[j]]; WP al=(WP)a,ale=left_end(a); // the child left words @@ -121,7 +125,6 @@ public: assert(anw<=M); // of course, we never store more than M left words in an item. } else { // single word WordID ew=e[j]; - FSAFFDBG(edge,' '<0;) { + bool sp=false; + for (int i=ctxlen_;i>0;sp=true) { --i; WordID w=wst[i]; if (w==TD::none) continue; - if (i) o<<' '; + if (sp) o<<' '; o << TD::Convert(w); } o<<']'; diff --git a/decoder/ff_lm_fsa.h b/decoder/ff_lm_fsa.h index 108698ec..9ba7b2c5 100755 --- a/decoder/ff_lm_fsa.h +++ b/decoder/ff_lm_fsa.h @@ -1,9 +1,10 @@ #ifndef FF_LM_FSA_H #define FF_LM_FSA_H -//FIXME: when FSA_LM_PHRASE 1, 3gram has differences in 4th decimal digit, compared to regular ff_lm. this is USUALLY a bug (there's way more actual precision in there). this was with #define LM_FSA_SHORTEN_CONTEXT 1 and 0 (so it's not that). also, LM_FSA_SHORTEN_CONTEXT gives identical scores with FSA_LM_PHRASE 0 +//FIXME: when FSA_LM_PHRASE 1, 3gram fsa has differences, especially with unk words, in about the 4th decimal digit (about .05%), compared to regular ff_lm. this is USUALLY a bug (there's way more actual precision in there). this was with #define LM_FSA_SHORTEN_CONTEXT 1 and 0 (so it's not that). also, LM_FSA_SHORTEN_CONTEXT gives identical scores with FSA_LM_PHRASE 0 -#define FSA_LM_PHRASE 0 +// enabling for now - retest unigram+ more, solve above puzzle +#define FSA_LM_PHRASE 1 #define FSA_LM_DEBUG 0 #if FSA_LM_DEBUG @@ -42,7 +43,8 @@ struct LanguageModelFsa : public FsaFeatureFunctionBase { } template - void ScanAccum(SentenceMetadata const& /* smeta */,Hypergraph::Edge const& /* edge */,WordID w,void const* old_st,void *new_st,Accum *a) const { + void ScanAccum(SentenceMetadata const& /* smeta */,Hypergraph::Edge const& edge,WordID w,void const* old_st,void *new_st,Accum *a) const { + Hypergraph::Edge &de=(Hypergraph::Edge &)edge; if (!ctxlen_) { Add(floored(pimpl_->WordProb(w,&empty_context)),a); return; @@ -53,6 +55,8 @@ struct LanguageModelFsa : public FsaFeatureFunctionBase { state_copy(ctx,old_st); ctx[ctxlen_]=TD::none; // make this part of state? wastes space but saves copies. Featval p=floored(pimpl_->WordProb(w,ctx)); + FSALMDBG(de,"p("< { //FIXME: there is a bug in here somewhere, or else the 3gram LM we use gives different scores for phrases (impossible? BOW nonzero when shortening context past what LM has?) template void ScanPhraseAccum(SentenceMetadata const& /* smeta */,const Hypergraph::Edge&edge,WordID const* begin,WordID const* end,void const* old_st,void *new_st,Accum *a) const { + Hypergraph::Edge &de=(Hypergraph::Edge &)edge; if (begin==end) return; // otherwise w/ shortening it's possible to end up with no words at all. /* // this is forcing unigram prob always. we will instead build the phrase if (!ctxlen_) { @@ -85,27 +90,29 @@ struct LanguageModelFsa : public FsaFeatureFunctionBase { ctx[nboth]=TD::none; // reverse order - state at very end of context, then [i,end) in rev order ending at ctx[0] W ctx_score_end=wordcpy_reverse(ctx,begin,end); - assert(ctx_score_end==ctx+nw); wordcpy(ctx_score_end,st,st_end); // st already reversed. + assert(ctx_score_end==ctx+nw); // we could just copy the filled state words, but it probably doesn't save much time (and might cost some to scan to find the nones. most contexts are full except for the shortest source spans. -// FSALMDBG(edge," Scan("<l("<l("<ctx;--ctx_score_end) p+=floored(pimpl_->WordProb(ctx_score_end[-1],ctx_score_end)); //TODO: look for score discrepancy - - // i had some idea that maybe shortencontext would return a different prob if the length provided was > ctxlen_; however, since the same 4th digit disagreement happens with LM_FSA_SHORTEN_CONTEXT 0 anyway, it's not that. perhaps look to SCAN_PHRASE_ACCUM_OVERRIDE - make sure they do the right thing. + // i had some idea that maybe shortencontext would return a different prob if the length provided was > ctxlen_; however, since the same disagreement happens with LM_FSA_SHORTEN_CONTEXT 0 anyway, it's not that. perhaps look to SCAN_PHRASE_ACCUM_OVERRIDE - make sure they do the right thing. #if LM_FSA_SHORTEN_CONTEXT p+=pimpl_->ShortenContext(ctx,nboth