From 6ca64b5cf2d4c0c2698d4298e88dde274f766ac4 Mon Sep 17 00:00:00 2001 From: graehl Date: Fri, 13 Aug 2010 03:30:49 +0000 Subject: named_enum, itoa, cdec replace --a-b=x with --a_b=x git-svn-id: https://ws10smt.googlecode.com/svn/trunk@536 ec762483-ff6d-05da-a07a-a48fb63a330f --- decoder/apply_fsa_models.cc | 20 ++++- decoder/apply_fsa_models.h | 27 ++++-- decoder/cdec.cc | 2 +- decoder/cfg.cc | 22 +++-- decoder/cfg.h | 16 ++-- decoder/cfg_options.h | 1 + decoder/program_options.h | 25 ++++++ utils/static_utoa.h | 24 +----- utils/utoa.h | 199 ++++++++++++++++++++++++++++++++++---------- 9 files changed, 245 insertions(+), 91 deletions(-) diff --git a/decoder/apply_fsa_models.cc b/decoder/apply_fsa_models.cc index 1c30eb90..2854b28b 100755 --- a/decoder/apply_fsa_models.cc +++ b/decoder/apply_fsa_models.cc @@ -13,6 +13,8 @@ using namespace std; +DEFINE_NAMED_ENUM(FSA_BY) + struct ApplyFsa { ApplyFsa(HgCFG &i, const SentenceMetadata& smeta, @@ -74,6 +76,7 @@ void ApplyFsaModels(HgCFG &i, a.Compute(); } +/* namespace { char const* anames[]={ "BU_CUBE", @@ -82,14 +85,18 @@ char const* anames[]={ 0 }; } +*/ //TODO: named enum type in boost? std::string ApplyFsaBy::name() const { - return anames[algorithm]; +// return anames[algorithm]; + return GetName(algorithm); } std::string ApplyFsaBy::all_names() { + return FsaByNames(" "); + /* std::ostringstream o; for (int i=0;i=N_ALGORITHMS) +ApplyFsaBy::ApplyFsaBy(FsaBy i, int pop_limit) : pop_limit(pop_limit) { +/* if (i<0 || i>=N_ALGORITHMS) throw std::runtime_error("Unknown ApplyFsaBy type id: "+itos(i)+" - legal types: "+all_names()); +*/ + GetName(i); // checks validity algorithm=i; } diff --git a/decoder/apply_fsa_models.h b/decoder/apply_fsa_models.h index 5120fb4e..6561c70c 100755 --- a/decoder/apply_fsa_models.h +++ b/decoder/apply_fsa_models.h @@ -4,25 +4,36 @@ #include #include #include "feature_vector.h" +#include "named_enum.h" struct FsaFeatureFunction; struct Hypergraph; struct SentenceMetadata; struct HgCFG; + +#define FSA_BY(X,t) \ + X(t,BU_CUBE,) \ + X(t,BU_FULL,) \ + X(t,EARLEY,) \ + +#define FSA_BY_TYPE FsaBy + +DECLARE_NAMED_ENUM(FSA_BY) + struct ApplyFsaBy { - enum { - BU_CUBE, - BU_FULL, - EARLEY, - N_ALGORITHMS - }; +/*enum { + BU_CUBE, + BU_FULL, + EARLEY, + N_ALGORITHMS + };*/ int pop_limit; // only applies to BU_FULL so far bool IsBottomUp() const { return algorithm==BU_FULL || algorithm==BU_CUBE; } int BottomUpAlgorithm() const; - int algorithm; + FsaBy algorithm; std::string name() const; friend inline std::ostream &operator << (std::ostream &o,ApplyFsaBy const& c) { o << c.name(); @@ -30,7 +41,7 @@ struct ApplyFsaBy { o << "("<::min(),std::numeric_limits::min()); -} - -WordID CFG::BinName(BinRhs const& b) +// index i >= N.size()? then it's in M[i-N.size()] +WordID BinName(CFG::BinRhs const& b,CFG::NTs const& N,CFG::NTs const& M) { + int nn=N.size(); ostringstream o; -#define BinNameOWORD(w) do { int n=w; if (n>0) o << TD::Convert(n); else { o << 'V' << -n; } } while(0) +#define BinNameOWORD(w) \ + do { \ + int n=w; if (n>0) o << TD::Convert(n); \ + else { \ + int i=-n; \ + CFG::NT const&nt = i Ruleids; void print_nt_name(std::ostream &o,NTHandle n) const { - o << nts[n].from; + o << nts[n].from << n; } typedef std::pair BinRhs; - WordID BinName(BinRhs const& b); struct Rule { // for binarizing - no costs/probs @@ -106,16 +105,17 @@ struct CFG { swap(goal_nt,o.goal_nt); } void Binarize(CFGBinarize const& binarize_options); + + typedef std::vector NTs; + NTs nts; + typedef std::vector Rules; + Rules rules; + int goal_nt; + prob_t goal_inside,pushed_inside; // when we push viterbi weights to goal, we store the removed probability in pushed_inside protected: bool uninit; Hypergraph const* hg_; // shouldn't be used for anything, esp. after binarization - prob_t goal_inside,pushed_inside; // when we push viterbi weights to goal, we store the removed probability in pushed_inside // rules/nts will have same index as hg edges/nodes - typedef std::vector Rules; - Rules rules; - typedef std::vector NTs; - NTs nts; - int goal_nt; }; inline void swap(CFG &a,CFG &b) { diff --git a/decoder/cfg_options.h b/decoder/cfg_options.h index 956586f0..acd8d05b 100755 --- a/decoder/cfg_options.h +++ b/decoder/cfg_options.h @@ -28,6 +28,7 @@ struct CFGOptions { void Validate() { format.Validate(); binarize.Validate(); +// if (cfg_output.empty()) binarize.bin_name_nts=false; } char const* description() const { return "CFG output options"; diff --git a/decoder/program_options.h b/decoder/program_options.h index 251f5680..87afb320 100755 --- a/decoder/program_options.h +++ b/decoder/program_options.h @@ -13,6 +13,31 @@ #include +// change --opt-name=x --opt_name=x for all strings x. danger: probably the argv from int main isn't supposed to be modified? +inline int arg_minusto_underscore(char *s) { + if (!*s || *s++ != '-') return 0; + if (!*s || *s++ != '-') return 0; + int chars_replaced=0; + for(;*s;++s) { + if (*s=='=') + break; + if (*s=='-') { + *s='_'; + ++chars_replaced; + } + } + return chars_replaced; +} + +inline +int argv_minus_to_underscore(int argc, char **argv) { + int chars_replaced=0; + for (int i=1;i boost::program_options::typed_value* defaulted_value(T *v) diff --git a/utils/static_utoa.h b/utils/static_utoa.h index 3af9fbb6..d15ed35b 100755 --- a/utils/static_utoa.h +++ b/utils/static_utoa.h @@ -5,35 +5,19 @@ #include "utoa.h" namespace { +static const int utoa_bufsize=40; // 64bit safe. +static const int utoa_bufsizem1=utoa_bufsize-1; // 64bit safe. THREADLOCAL char utoa_buf[utoa_bufsize]; // to put end of string character at buf[20] } inline char *static_utoa(unsigned n) { + assert(utoa_buf[utoa_bufsizem1]==0); return utoa(utoa_buf+utoa_bufsizem1,n); } inline char *static_itoa(int n) { + assert(utoa_buf[utoa_bufsizem1]==0); return itoa(utoa_buf+utoa_bufsizem1,n); } -#ifdef ITOA_SAMPLE -# include -# include -# include -using namespace std; - -int main(int argc,char *argv[]) { - printf("d U d U d U\n"); - for (int i=1;i #include #include +#include +// define this if you're paranoid about converting 0-9 (int) to 0-9 (char) by adding to '0', which is safe for ascii, utf8, etc. #ifndef DIGIT_LOOKUP_TABLE # define DIGIT_LOOKUP_TABLE 0 #endif +//TODO: 3 decimal digit lookup table, divide by 1000 faster? +//TODO: benchmark these two (also, some assembly that does effectively divmod?) +#if 1 +// maybe this is faster than mod because we are already dividing +#define NDIV10MOD(rem,n) rem = n; n /= 10; rem -= 10*n; +#else +// or maybe optimizer does it just as well or better with this: +#define NDIV10MOD(rem,n) rem = n%10; n = n/10; +#endif + +template +struct signed_for_int { +}; + + +#define DEFINE_SIGNED_FOR_3(t,it,ut) \ +template <> \ +struct signed_for_int { \ + typedef ut unsigned_t; \ + typedef it signed_t; \ + typedef t original_t; \ + enum { toa_bufsize = 3 + std::numeric_limits::digits10, toa_bufsize_minus_1=toa_bufsize-1 }; \ +}; + +// toa_bufsize will hold enough chars for a c string converting to sign,digits (for both signed and unsigned types), because normally an unsigned would only need 2 extra chars. we reserve 3 explicitly for the case that itoa(buf,UINT_MAX,true) is called, with output +4...... + +#define DEFINE_SIGNED_FOR(it) DEFINE_SIGNED_FOR_3(it,it,u ## it) \ + DEFINE_SIGNED_FOR_3(u ## it,it,u ## it) + +DEFINE_SIGNED_FOR(int8_t) +DEFINE_SIGNED_FOR(int16_t) +DEFINE_SIGNED_FOR(int32_t) +DEFINE_SIGNED_FOR(int64_t) +//DEFINE_SIGNED_FOR_3(int,int,unsigned) +//DEFINE_SIGNED_FOR_3(unsigned,int,unsigned) + +/* // The largest 32-bit integer is 4294967295, that is 10 chars // 1 more for sign, and 1 for 0-termination of string -// generally: 2 + std::numeric_limits::is_signed + std::numeric_limits::digits10 const unsigned utoa_bufsize=12; const unsigned utoa_bufsizem1=utoa_bufsize-1; +const unsigned ultoa_bufsize=22; +const unsigned ultoa_bufsizem1=utoa_bufsize-1; +*/ #ifdef DIGIT_LOOKUP_TABLE namespace { @@ -30,72 +72,139 @@ inline char digit_to_char(int d) { #endif } -// returns n in string [return,num); *num=0 yourself before calling if you want a c_str -inline char *utoa(char *num,uint32_t n) { - if ( !n ) { - *--num='0'; + +// returns n in string [return,num); *num=0 yourself before calling if you want a c_str. in other words, the sequence [ret,buf) contains the written digits +template +char *utoa(char *buf,Int n_) { + typedef typename signed_for_int::unsigned_t Uint; + Uint n=n_; + if (!n) { + *--buf='0'; } else { - uint32_t rem; + Uint rem; // 3digit lookup table, divide by 1000 faster? - while ( n ) { -#if 1 - rem = n; - n /= 10; - rem -= 10*n; // maybe this is faster than mod because we are already dividing -#else - rem = n%10; // would optimizer combine these together? - n = n/10; -#endif - *--num = digit_to_char(rem); + while (n) { + NDIV10MOD(rem,n); + *--buf = digit_to_char(rem); } } - return num; + return buf; } -inline char *itoa(char *p,int32_t n) { - if (n<0) { - p=utoa(p,-n); // (unsigned)(-INT_MIN) == 0x1000000 in 2s complement and not == 0. - *--p='-'; - return p; - } else - return utoa(p,n); +// left_pad_0(buf,utoa(buf+bufsz,n)) means that [buf,buf+bufsz) is a left-0 padded seq. of digits. no 0s are added if utoa is already past buf (you must have ensured that this is valid memory, naturally) +inline void left_pad(char *left,char *buf,char pad='0') { + while (buf>left) + *--buf=pad; + // return buf; } -inline std::string utos(uint32_t n) { - char buf[utoa_bufsize]; - char *end=buf+utoa_bufsize; - char *p=utoa(end,n); +template +char *utoa_left_pad(char *buf,char *bufend,Int n, char pad='0') { + char *r=utoa(bufend,n); + assert(buf<=r); + left_pad(buf,r,pad); + return buf; +} + +// note: 0 -> 0, but otherwise x000000 -> x (x has no trailing 0s). same conditions as utoa; [ret,buf) gives the sequence of digits +// useful for floating point fraction output +template +char *utoa_drop_trailing_0(char *buf,Uint_ n_, unsigned &n_skipped) { + typedef typename signed_for_int::unsigned_t Uint; + Uint n=n_; + n_skipped=0; + if (!n) { + *--buf='0'; + return buf; + } else { + Uint rem; + while (n) { + NDIV10MOD(rem,n); + if (rem) { + *--buf = digit_to_char(rem); + // some non-0 trailing digits; now output all digits. + while (n) { + NDIV10MOD(rem,n); + *--buf = digit_to_char(rem); + } + return buf; + } + ++n_skipped; + } + assert(0); + return 0; + } +} + +// desired feature: itoa(unsigned) = utoa(unsigned) +// positive sign: 0 -> +0, 1-> +1. obviously -n -> -n +template +//typename signed_for_int::original_t instead of Int to give more informative wrong-type message? +char *itoa(char *buf,Int i,bool positive_sign=false) { + typename signed_for_int::unsigned_t n=i; + if (i<0) + n=-n; //sidesteps 2s complement issue doing this rather than just u=-n. + char * ret=utoa(buf,n); + if (i<0) { + *--ret='-'; + } else if (positive_sign) + *--ret='+'; + return ret; +} + +template +char * itoa_left_pad(char *buf,char *bufend,Int i,bool positive_sign=false,char pad='0') { + typename signed_for_int::unsigned_t n=i; + if (i<0) { + n=-n; //sidesteps 2s complement issue doing this rather than just u=-n. + *buf='-'; + } else if (positive_sign) + *buf='+'; + char * r=utoa(bufend,n); + assert(buf +inline std::string itos(Int n) { + char buf[signed_for_int::toa_bufsize]; + char *end=buf+signed_for_int::toa_bufsize; + char *p=itoa(end,n); return std::string(p,end); } -inline std::string itos(int32_t n) { - char buf[utoa_bufsize]; - char *end=buf+utoa_bufsize; +template +inline std::string utos(Int n) { + char buf[signed_for_int::toa_bufsize]; + char *end=buf+signed_for_int::toa_bufsize; char *p=itoa(end,n); return std::string(p,end); } //returns position of '\0' terminating number written starting at to -inline char* append_utoa(char *to,uint32_t n) { - char buf[utoa_bufsize]; - char *end=buf+utoa_bufsize; - char *s=utoa(end,n); - int ns=end-s; - std::memcpy(to,s,ns); +template +inline char* append_utoa(char *to,typename signed_for_int::unsigned_t n) { + char buf[signed_for_int::toa_bufsize]; + char *end=buf+signed_for_int::toa_bufsize; + char *p=itoa(end,n); + int ns=end-p; + std::memcpy(to,p,ns); to+=ns; - *to++=0; + *to=0; return to; } //returns position of '\0' terminating number written starting at to -inline char* append_itoa(char *to,int32_t n) { - char buf[utoa_bufsize]; - char *end=buf+utoa_bufsize; - char *s=itoa(end,n); - int ns=end-s; - std::memcpy(to,s,ns); +template +inline char* append_itoa(char *to,typename signed_for_int::signed_t n) { + char buf[signed_for_int::toa_bufsize]; + char *end=buf+signed_for_int::toa_bufsize; + char *p=utoa(end,n); + int ns=end-p; + std::memcpy(to,p,ns); to+=ns; - *to++=0; + *to=0; return to; } -- cgit v1.2.3