summaryrefslogtreecommitdiff
path: root/decoder/tdict.h
diff options
context:
space:
mode:
authorgraehl <graehl@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-07-21 22:09:50 +0000
committergraehl <graehl@ec762483-ff6d-05da-a07a-a48fb63a330f>2010-07-21 22:09:50 +0000
commitdd456888d9648ff8a79c0eee36c03a4dc5558b1c (patch)
tree76d74b02bf1f1fc5ec7a991c0ba08846390eeff9 /decoder/tdict.h
parentcd2264b189a732f9252492118af373f8c60e2f25 (diff)
disabled TD reserved stuff - debug init assertion later
git-svn-id: https://ws10smt.googlecode.com/svn/trunk@364 ec762483-ff6d-05da-a07a-a48fb63a330f
Diffstat (limited to 'decoder/tdict.h')
-rw-r--r--decoder/tdict.h8
1 files changed, 5 insertions, 3 deletions
diff --git a/decoder/tdict.h b/decoder/tdict.h
index 26e94edf..e0b4b5f0 100644
--- a/decoder/tdict.h
+++ b/decoder/tdict.h
@@ -9,21 +9,23 @@
class Vocab;
struct TD {
+ /* // disabled for now
static const int reserved_begin=10; // allow room for SRI special tokens e.g. unk ss se pause. tokens until this get "<FILLERi>"
static const int n_reserved=10; // 0...n_reserved-1 get token '<RESERVEDi>'
static inline WordID reserved(int i) {
assert(i>=0 && i<n_reserved);
return (WordID)(reserved_begin+i);
}
+ static inline WordID begin() {
+ return reserved(n_reserved);
+ }
+ */
static const WordID max_wordid=0x7fffffff;
static const WordID none=(WordID)-1; // Vocab_None
static char const* const ss_str; //="<s>";
static char const* const se_str; //="</s>";
static char const* const unk_str; //="<unk>";
static WordID ss,se,unk; // x=Convert(x_str)
- static inline WordID begin() {
- return reserved(n_reserved);
- }
static WordID end(); // next id to be assigned; [begin,end) give the non-reserved tokens seen so far
static Vocab dict_;
static void ConvertSentence(std::string const& sent, std::vector<WordID>* ids);