summaryrefslogtreecommitdiff
path: root/decoder/tdict.h
diff options
context:
space:
mode:
Diffstat (limited to 'decoder/tdict.h')
-rw-r--r--decoder/tdict.h50
1 files changed, 0 insertions, 50 deletions
diff --git a/decoder/tdict.h b/decoder/tdict.h
deleted file mode 100644
index a7b3ee1c..00000000
--- a/decoder/tdict.h
+++ /dev/null
@@ -1,50 +0,0 @@
-#ifndef _TDICT_H_
-#define _TDICT_H_
-
-#include <string>
-#include <vector>
-#include "wordid.h"
-#include <assert.h>
-
-class Vocab;
-
-struct TD {
- /* // disabled for now
- static const int reserved_begin=10; // allow room for SRI special tokens e.g. unk ss se pause. tokens until this get "<FILLERi>"
- static const int n_reserved=10; // 0...n_reserved-1 get token '<RESERVEDi>'
- static inline WordID reserved(int i) {
- assert(i>=0 && i<n_reserved);
- return (WordID)(reserved_begin+i);
- }
- static inline WordID begin() {
- return reserved(n_reserved);
- }
- */
- static const WordID max_wordid=0x7fffffff;
- static const WordID none=(WordID)-1; // Vocab_None
- static char const* const ss_str; //="<s>";
- static char const* const se_str; //="</s>";
- static char const* const unk_str; //="<unk>";
- static WordID ss,se,unk; // x=Convert(x_str)
- static WordID end(); // next id to be assigned; [begin,end) give the non-reserved tokens seen so far
- static Vocab dict_;
- static void ConvertSentence(std::string const& sent, std::vector<WordID>* ids);
- static void GetWordIDs(const std::vector<std::string>& strings, std::vector<WordID>* ids);
- static std::string GetString(const std::vector<WordID>& str);
- static std::string GetString(WordID const* i,WordID const* e);
- static int AppendString(const WordID& w, int pos, int bufsize, char* buffer);
- static unsigned int NumWords();
- static WordID Convert(const std::string& s);
- static WordID Convert(char const* s);
- static const char* Convert(WordID w);
-};
-
-struct ToTD {
- typedef WordID result_type;
- result_type operator()(std::string const& t) const {
- return TD::Convert(t);
- }
-};
-
-
-#endif