diff options
Diffstat (limited to 'decoder/tdict.h')
| -rw-r--r-- | decoder/tdict.h | 8 |
1 files changed, 5 insertions, 3 deletions
diff --git a/decoder/tdict.h b/decoder/tdict.h index 26e94edf..e0b4b5f0 100644 --- a/decoder/tdict.h +++ b/decoder/tdict.h @@ -9,21 +9,23 @@ class Vocab; struct TD { + /* // disabled for now static const int reserved_begin=10; // allow room for SRI special tokens e.g. unk ss se pause. tokens until this get "<FILLERi>" static const int n_reserved=10; // 0...n_reserved-1 get token '<RESERVEDi>' static inline WordID reserved(int i) { assert(i>=0 && i<n_reserved); return (WordID)(reserved_begin+i); } + static inline WordID begin() { + return reserved(n_reserved); + } + */ static const WordID max_wordid=0x7fffffff; static const WordID none=(WordID)-1; // Vocab_None static char const* const ss_str; //="<s>"; static char const* const se_str; //="</s>"; static char const* const unk_str; //="<unk>"; static WordID ss,se,unk; // x=Convert(x_str) - static inline WordID begin() { - return reserved(n_reserved); - } static WordID end(); // next id to be assigned; [begin,end) give the non-reserved tokens seen so far static Vocab dict_; static void ConvertSentence(std::string const& sent, std::vector<WordID>* ids); |