summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xdecoder/apply_fsa_models.cc20
-rwxr-xr-xdecoder/apply_fsa_models.h27
-rw-r--r--decoder/cdec.cc2
-rwxr-xr-xdecoder/cfg.cc22
-rwxr-xr-xdecoder/cfg.h16
-rwxr-xr-xdecoder/cfg_options.h1
-rwxr-xr-xdecoder/program_options.h25
-rwxr-xr-xutils/static_utoa.h24
-rwxr-xr-xutils/utoa.h199
9 files changed, 245 insertions, 91 deletions
diff --git a/decoder/apply_fsa_models.cc b/decoder/apply_fsa_models.cc
index 1c30eb90..2854b28b 100755
--- a/decoder/apply_fsa_models.cc
+++ b/decoder/apply_fsa_models.cc
@@ -13,6 +13,8 @@
using namespace std;
+DEFINE_NAMED_ENUM(FSA_BY)
+
struct ApplyFsa {
ApplyFsa(HgCFG &i,
const SentenceMetadata& smeta,
@@ -74,6 +76,7 @@ void ApplyFsaModels(HgCFG &i,
a.Compute();
}
+/*
namespace {
char const* anames[]={
"BU_CUBE",
@@ -82,14 +85,18 @@ char const* anames[]={
0
};
}
+*/
//TODO: named enum type in boost?
std::string ApplyFsaBy::name() const {
- return anames[algorithm];
+// return anames[algorithm];
+ return GetName(algorithm);
}
std::string ApplyFsaBy::all_names() {
+ return FsaByNames(" ");
+ /*
std::ostringstream o;
for (int i=0;i<N_ALGORITHMS;++i) {
assert(anames[i]);
@@ -97,19 +104,24 @@ std::string ApplyFsaBy::all_names() {
o<<anames[i];
}
return o.str();
+ */
}
ApplyFsaBy::ApplyFsaBy(std::string const& n, int pop_limit) : pop_limit(pop_limit) {
- algorithm=0;
std::string uname=toupper(n);
+ algorithm=GetFsaBy(uname);
+/*anames=0;
while(anames[algorithm] && anames[algorithm] != uname) ++algorithm;
if (!anames[algorithm])
throw std::runtime_error("Unknown ApplyFsaBy type: "+n+" - legal types: "+all_names());
+*/
}
-ApplyFsaBy::ApplyFsaBy(int i, int pop_limit) : pop_limit(pop_limit) {
- if (i<0 || i>=N_ALGORITHMS)
+ApplyFsaBy::ApplyFsaBy(FsaBy i, int pop_limit) : pop_limit(pop_limit) {
+/* if (i<0 || i>=N_ALGORITHMS)
throw std::runtime_error("Unknown ApplyFsaBy type id: "+itos(i)+" - legal types: "+all_names());
+*/
+ GetName(i); // checks validity
algorithm=i;
}
diff --git a/decoder/apply_fsa_models.h b/decoder/apply_fsa_models.h
index 5120fb4e..6561c70c 100755
--- a/decoder/apply_fsa_models.h
+++ b/decoder/apply_fsa_models.h
@@ -4,25 +4,36 @@
#include <string>
#include <iostream>
#include "feature_vector.h"
+#include "named_enum.h"
struct FsaFeatureFunction;
struct Hypergraph;
struct SentenceMetadata;
struct HgCFG;
+
+#define FSA_BY(X,t) \
+ X(t,BU_CUBE,) \
+ X(t,BU_FULL,) \
+ X(t,EARLEY,) \
+
+#define FSA_BY_TYPE FsaBy
+
+DECLARE_NAMED_ENUM(FSA_BY)
+
struct ApplyFsaBy {
- enum {
- BU_CUBE,
- BU_FULL,
- EARLEY,
- N_ALGORITHMS
- };
+/*enum {
+ BU_CUBE,
+ BU_FULL,
+ EARLEY,
+ N_ALGORITHMS
+ };*/
int pop_limit; // only applies to BU_FULL so far
bool IsBottomUp() const {
return algorithm==BU_FULL || algorithm==BU_CUBE;
}
int BottomUpAlgorithm() const;
- int algorithm;
+ FsaBy algorithm;
std::string name() const;
friend inline std::ostream &operator << (std::ostream &o,ApplyFsaBy const& c) {
o << c.name();
@@ -30,7 +41,7 @@ struct ApplyFsaBy {
o << "("<<c.pop_limit<<")";
return o;
}
- explicit ApplyFsaBy(int alg, int poplimit=200);
+ explicit ApplyFsaBy(FsaBy alg, int poplimit=200);
ApplyFsaBy(std::string const& name, int poplimit=200);
ApplyFsaBy(const ApplyFsaBy &o) : algorithm(o.algorithm) { }
static std::string all_names(); // space separated
diff --git a/decoder/cdec.cc b/decoder/cdec.cc
index 3633febd..5898b245 100644
--- a/decoder/cdec.cc
+++ b/decoder/cdec.cc
@@ -193,7 +193,7 @@ void InitCommandLine(int argc, char** argv, OracleBleu &ob, po::variables_map* c
dconfig_options.add(opts).add(cfgo);
//add(opts).add(cfgo)
dcmdline_options.add(dconfig_options).add(clo);
-
+ argv_minus_to_underscore(argc,argv);
po::store(parse_command_line(argc, argv, dcmdline_options), conf);
if (conf.count("compgen")) {
print_options(cout,dcmdline_options);
diff --git a/decoder/cfg.cc b/decoder/cfg.cc
index 81a17355..aa9e5f30 100755
--- a/decoder/cfg.cc
+++ b/decoder/cfg.cc
@@ -11,13 +11,21 @@ using namespace std;
namespace {
CFG::BinRhs nullrhs(std::numeric_limits<int>::min(),std::numeric_limits<int>::min());
-}
-
-WordID CFG::BinName(BinRhs const& b)
+// index i >= N.size()? then it's in M[i-N.size()]
+WordID BinName(CFG::BinRhs const& b,CFG::NTs const& N,CFG::NTs const& M)
{
+ int nn=N.size();
ostringstream o;
-#define BinNameOWORD(w) do { int n=w; if (n>0) o << TD::Convert(n); else { o << 'V' << -n; } } while(0)
+#define BinNameOWORD(w) \
+ do { \
+ int n=w; if (n>0) o << TD::Convert(n); \
+ else { \
+ int i=-n; \
+ CFG::NT const&nt = i<nn?N[i]:M[i-nn]; \
+ o << nt.from << i; } \
+ } while(0)
+
BinNameOWORD(b.first);
o<<'+';
BinNameOWORD(b.second);
@@ -25,6 +33,10 @@ WordID CFG::BinName(BinRhs const& b)
return TD::Convert(o.str());
}
+}
+
+
+
void CFG::Binarize(CFGBinarize const& b) {
if (!b.Binarizing()) return;
if (!b.bin_l2r) {
@@ -57,7 +69,7 @@ void CFG::Binarize(CFGBinarize const& b) {
new_nts.back().ruleids.push_back(newruleid);
new_rules.push_back(Rule(newnt,bin));
if (b.bin_name_nts)
- new_nts.back().from.nt=BinName(bin);
+ new_nts.back().from.nt=BinName(bin,nts,new_nts);
++newnt;++newruleid;
}
}
diff --git a/decoder/cfg.h b/decoder/cfg.h
index 64924f14..e1f818e8 100755
--- a/decoder/cfg.h
+++ b/decoder/cfg.h
@@ -40,11 +40,10 @@ struct CFG {
typedef std::vector<RuleHandle> Ruleids;
void print_nt_name(std::ostream &o,NTHandle n) const {
- o << nts[n].from;
+ o << nts[n].from << n;
}
typedef std::pair<int,int> BinRhs;
- WordID BinName(BinRhs const& b);
struct Rule {
// for binarizing - no costs/probs
@@ -106,16 +105,17 @@ struct CFG {
swap(goal_nt,o.goal_nt);
}
void Binarize(CFGBinarize const& binarize_options);
+
+ typedef std::vector<NT> NTs;
+ NTs nts;
+ typedef std::vector<Rule> Rules;
+ Rules rules;
+ int goal_nt;
+ prob_t goal_inside,pushed_inside; // when we push viterbi weights to goal, we store the removed probability in pushed_inside
protected:
bool uninit;
Hypergraph const* hg_; // shouldn't be used for anything, esp. after binarization
- prob_t goal_inside,pushed_inside; // when we push viterbi weights to goal, we store the removed probability in pushed_inside
// rules/nts will have same index as hg edges/nodes
- typedef std::vector<Rule> Rules;
- Rules rules;
- typedef std::vector<NT> NTs;
- NTs nts;
- int goal_nt;
};
inline void swap(CFG &a,CFG &b) {
diff --git a/decoder/cfg_options.h b/decoder/cfg_options.h
index 956586f0..acd8d05b 100755
--- a/decoder/cfg_options.h
+++ b/decoder/cfg_options.h
@@ -28,6 +28,7 @@ struct CFGOptions {
void Validate() {
format.Validate();
binarize.Validate();
+// if (cfg_output.empty()) binarize.bin_name_nts=false;
}
char const* description() const {
return "CFG output options";
diff --git a/decoder/program_options.h b/decoder/program_options.h
index 251f5680..87afb320 100755
--- a/decoder/program_options.h
+++ b/decoder/program_options.h
@@ -13,6 +13,31 @@
#include <iosfwd>
+// change --opt-name=x --opt_name=x for all strings x. danger: probably the argv from int main isn't supposed to be modified?
+inline int arg_minusto_underscore(char *s) {
+ if (!*s || *s++ != '-') return 0;
+ if (!*s || *s++ != '-') return 0;
+ int chars_replaced=0;
+ for(;*s;++s) {
+ if (*s=='=')
+ break;
+ if (*s=='-') {
+ *s='_';
+ ++chars_replaced;
+ }
+ }
+ return chars_replaced;
+}
+
+inline
+int argv_minus_to_underscore(int argc, char **argv) {
+ int chars_replaced=0;
+ for (int i=1;i<argc;++i) {
+ chars_replaced+=arg_minusto_underscore(argv[i]);
+ }
+ return chars_replaced;
+}
+
template <class T>
boost::program_options::typed_value<T>*
defaulted_value(T *v)
diff --git a/utils/static_utoa.h b/utils/static_utoa.h
index 3af9fbb6..d15ed35b 100755
--- a/utils/static_utoa.h
+++ b/utils/static_utoa.h
@@ -5,35 +5,19 @@
#include "utoa.h"
namespace {
+static const int utoa_bufsize=40; // 64bit safe.
+static const int utoa_bufsizem1=utoa_bufsize-1; // 64bit safe.
THREADLOCAL char utoa_buf[utoa_bufsize]; // to put end of string character at buf[20]
}
inline char *static_utoa(unsigned n) {
+ assert(utoa_buf[utoa_bufsizem1]==0);
return utoa(utoa_buf+utoa_bufsizem1,n);
}
inline char *static_itoa(int n) {
+ assert(utoa_buf[utoa_bufsizem1]==0);
return itoa(utoa_buf+utoa_bufsizem1,n);
}
-#ifdef ITOA_SAMPLE
-# include <cstdio>
-# include <sstream>
-# include <iostream>
-using namespace std;
-
-int main(int argc,char *argv[]) {
- printf("d U d U d U\n");
- for (int i=1;i<argc;++i) {
- int n;
- unsigned un;
- sscanf(argv[i],"%d",&n);
- sscanf(argv[i],"%u",&un);
- printf("%d %u %s",n,un,static_itoa(n));
- printf(" %s %s %s\n",static_utoa(un),itos(n).c_str(),utos(un).c_str());
- }
- return 0;
-}
-#endif
-
#endif
diff --git a/utils/utoa.h b/utils/utoa.h
index 8d304f97..341965cc 100755
--- a/utils/utoa.h
+++ b/utils/utoa.h
@@ -1,19 +1,61 @@
#ifndef UTOA_H
#define UTOA_H
+
#include <stdint.h>
#include <string>
#include <cstring>
+#include <limits>
+// define this if you're paranoid about converting 0-9 (int) to 0-9 (char) by adding to '0', which is safe for ascii, utf8, etc.
#ifndef DIGIT_LOOKUP_TABLE
# define DIGIT_LOOKUP_TABLE 0
#endif
+//TODO: 3 decimal digit lookup table, divide by 1000 faster?
+//TODO: benchmark these two (also, some assembly that does effectively divmod?)
+#if 1
+// maybe this is faster than mod because we are already dividing
+#define NDIV10MOD(rem,n) rem = n; n /= 10; rem -= 10*n;
+#else
+// or maybe optimizer does it just as well or better with this:
+#define NDIV10MOD(rem,n) rem = n%10; n = n/10;
+#endif
+
+template <class T>
+struct signed_for_int {
+};
+
+
+#define DEFINE_SIGNED_FOR_3(t,it,ut) \
+template <> \
+struct signed_for_int<t> { \
+ typedef ut unsigned_t; \
+ typedef it signed_t; \
+ typedef t original_t; \
+ enum { toa_bufsize = 3 + std::numeric_limits<t>::digits10, toa_bufsize_minus_1=toa_bufsize-1 }; \
+};
+
+// toa_bufsize will hold enough chars for a c string converting to sign,digits (for both signed and unsigned types), because normally an unsigned would only need 2 extra chars. we reserve 3 explicitly for the case that itoa(buf,UINT_MAX,true) is called, with output +4......
+
+#define DEFINE_SIGNED_FOR(it) DEFINE_SIGNED_FOR_3(it,it,u ## it) \
+ DEFINE_SIGNED_FOR_3(u ## it,it,u ## it)
+
+DEFINE_SIGNED_FOR(int8_t)
+DEFINE_SIGNED_FOR(int16_t)
+DEFINE_SIGNED_FOR(int32_t)
+DEFINE_SIGNED_FOR(int64_t)
+//DEFINE_SIGNED_FOR_3(int,int,unsigned)
+//DEFINE_SIGNED_FOR_3(unsigned,int,unsigned)
+
+/*
// The largest 32-bit integer is 4294967295, that is 10 chars
// 1 more for sign, and 1 for 0-termination of string
-// generally: 2 + std::numeric_limits<T>::is_signed + std::numeric_limits<T>::digits10
const unsigned utoa_bufsize=12;
const unsigned utoa_bufsizem1=utoa_bufsize-1;
+const unsigned ultoa_bufsize=22;
+const unsigned ultoa_bufsizem1=utoa_bufsize-1;
+*/
#ifdef DIGIT_LOOKUP_TABLE
namespace {
@@ -30,72 +72,139 @@ inline char digit_to_char(int d) {
#endif
}
-// returns n in string [return,num); *num=0 yourself before calling if you want a c_str
-inline char *utoa(char *num,uint32_t n) {
- if ( !n ) {
- *--num='0';
+
+// returns n in string [return,num); *num=0 yourself before calling if you want a c_str. in other words, the sequence [ret,buf) contains the written digits
+template <class Int>
+char *utoa(char *buf,Int n_) {
+ typedef typename signed_for_int<Int>::unsigned_t Uint;
+ Uint n=n_;
+ if (!n) {
+ *--buf='0';
} else {
- uint32_t rem;
+ Uint rem;
// 3digit lookup table, divide by 1000 faster?
- while ( n ) {
-#if 1
- rem = n;
- n /= 10;
- rem -= 10*n; // maybe this is faster than mod because we are already dividing
-#else
- rem = n%10; // would optimizer combine these together?
- n = n/10;
-#endif
- *--num = digit_to_char(rem);
+ while (n) {
+ NDIV10MOD(rem,n);
+ *--buf = digit_to_char(rem);
}
}
- return num;
+ return buf;
}
-inline char *itoa(char *p,int32_t n) {
- if (n<0) {
- p=utoa(p,-n); // (unsigned)(-INT_MIN) == 0x1000000 in 2s complement and not == 0.
- *--p='-';
- return p;
- } else
- return utoa(p,n);
+// left_pad_0(buf,utoa(buf+bufsz,n)) means that [buf,buf+bufsz) is a left-0 padded seq. of digits. no 0s are added if utoa is already past buf (you must have ensured that this is valid memory, naturally)
+inline void left_pad(char *left,char *buf,char pad='0') {
+ while (buf>left)
+ *--buf=pad;
+ // return buf;
}
-inline std::string utos(uint32_t n) {
- char buf[utoa_bufsize];
- char *end=buf+utoa_bufsize;
- char *p=utoa(end,n);
+template <class Int>
+char *utoa_left_pad(char *buf,char *bufend,Int n, char pad='0') {
+ char *r=utoa(bufend,n);
+ assert(buf<=r);
+ left_pad(buf,r,pad);
+ return buf;
+}
+
+// note: 0 -> 0, but otherwise x000000 -> x (x has no trailing 0s). same conditions as utoa; [ret,buf) gives the sequence of digits
+// useful for floating point fraction output
+template <class Uint_>
+char *utoa_drop_trailing_0(char *buf,Uint_ n_, unsigned &n_skipped) {
+ typedef typename signed_for_int<Uint_>::unsigned_t Uint;
+ Uint n=n_;
+ n_skipped=0;
+ if (!n) {
+ *--buf='0';
+ return buf;
+ } else {
+ Uint rem;
+ while (n) {
+ NDIV10MOD(rem,n);
+ if (rem) {
+ *--buf = digit_to_char(rem);
+ // some non-0 trailing digits; now output all digits.
+ while (n) {
+ NDIV10MOD(rem,n);
+ *--buf = digit_to_char(rem);
+ }
+ return buf;
+ }
+ ++n_skipped;
+ }
+ assert(0);
+ return 0;
+ }
+}
+
+// desired feature: itoa(unsigned) = utoa(unsigned)
+// positive sign: 0 -> +0, 1-> +1. obviously -n -> -n
+template <class Int>
+//typename signed_for_int<Int>::original_t instead of Int to give more informative wrong-type message?
+char *itoa(char *buf,Int i,bool positive_sign=false) {
+ typename signed_for_int<Int>::unsigned_t n=i;
+ if (i<0)
+ n=-n; //sidesteps 2s complement issue doing this rather than just u=-n.
+ char * ret=utoa(buf,n);
+ if (i<0) {
+ *--ret='-';
+ } else if (positive_sign)
+ *--ret='+';
+ return ret;
+}
+
+template <class Int>
+char * itoa_left_pad(char *buf,char *bufend,Int i,bool positive_sign=false,char pad='0') {
+ typename signed_for_int<Int>::unsigned_t n=i;
+ if (i<0) {
+ n=-n; //sidesteps 2s complement issue doing this rather than just u=-n.
+ *buf='-';
+ } else if (positive_sign)
+ *buf='+';
+ char * r=utoa(bufend,n);
+ assert(buf<r);
+ left_pad(buf+1,r,pad);
+ return buf;
+}
+
+template <class Int>
+inline std::string itos(Int n) {
+ char buf[signed_for_int<Int>::toa_bufsize];
+ char *end=buf+signed_for_int<Int>::toa_bufsize;
+ char *p=itoa(end,n);
return std::string(p,end);
}
-inline std::string itos(int32_t n) {
- char buf[utoa_bufsize];
- char *end=buf+utoa_bufsize;
+template <class Int>
+inline std::string utos(Int n) {
+ char buf[signed_for_int<Int>::toa_bufsize];
+ char *end=buf+signed_for_int<Int>::toa_bufsize;
char *p=itoa(end,n);
return std::string(p,end);
}
//returns position of '\0' terminating number written starting at to
-inline char* append_utoa(char *to,uint32_t n) {
- char buf[utoa_bufsize];
- char *end=buf+utoa_bufsize;
- char *s=utoa(end,n);
- int ns=end-s;
- std::memcpy(to,s,ns);
+template <class Int>
+inline char* append_utoa(char *to,typename signed_for_int<Int>::unsigned_t n) {
+ char buf[signed_for_int<Int>::toa_bufsize];
+ char *end=buf+signed_for_int<Int>::toa_bufsize;
+ char *p=itoa(end,n);
+ int ns=end-p;
+ std::memcpy(to,p,ns);
to+=ns;
- *to++=0;
+ *to=0;
return to;
}
//returns position of '\0' terminating number written starting at to
-inline char* append_itoa(char *to,int32_t n) {
- char buf[utoa_bufsize];
- char *end=buf+utoa_bufsize;
- char *s=itoa(end,n);
- int ns=end-s;
- std::memcpy(to,s,ns);
+template <class Int>
+inline char* append_itoa(char *to,typename signed_for_int<Int>::signed_t n) {
+ char buf[signed_for_int<Int>::toa_bufsize];
+ char *end=buf+signed_for_int<Int>::toa_bufsize;
+ char *p=utoa(end,n);
+ int ns=end-p;
+ std::memcpy(to,p,ns);
to+=ns;
- *to++=0;
+ *to=0;
return to;
}