diff options
| author | Wu, Ke <wuke@cs.umd.edu> | 2014-12-17 16:15:13 -0500 |
|---|---|---|
| committer | Wu, Ke <wuke@cs.umd.edu> | 2014-12-17 16:15:13 -0500 |
| commit | 6829a0bc624b02ebefc79f8cf9ec89d7d64a7c30 (patch) | |
| tree | 125dfb20f73342873476c793995397b26fd202dd /klm/util/tokenize_piece.hh | |
| parent | b455a108a21f4ba5a58ab1bc53a8d2bf4d829067 (diff) | |
| parent | 7468e8d85e99b4619442c7afaf4a0d92870111bb (diff) | |
Merge branch 'const_reorder_2' into softsyn_2
Diffstat (limited to 'klm/util/tokenize_piece.hh')
| -rw-r--r-- | klm/util/tokenize_piece.hh | 16 |
1 files changed, 12 insertions, 4 deletions
diff --git a/klm/util/tokenize_piece.hh b/klm/util/tokenize_piece.hh index 24eae8fb..908c8daf 100644 --- a/klm/util/tokenize_piece.hh +++ b/klm/util/tokenize_piece.hh @@ -1,5 +1,5 @@ -#ifndef UTIL_TOKENIZE_PIECE__ -#define UTIL_TOKENIZE_PIECE__ +#ifndef UTIL_TOKENIZE_PIECE_H +#define UTIL_TOKENIZE_PIECE_H #include "util/exception.hh" #include "util/string_piece.hh" @@ -7,7 +7,8 @@ #include <boost/iterator/iterator_facade.hpp> #include <algorithm> -#include <iostream> + +#include <string.h> namespace util { @@ -71,6 +72,13 @@ class BoolCharacter { return StringPiece(in.data() + in.size(), 0); } + template <unsigned Length> static void Build(const char (&characters)[Length], bool (&out)[256]) { + memset(out, 0, sizeof(out)); + for (const char *i = characters; i != characters + Length; ++i) { + out[static_cast<unsigned char>(*i)] = true; + } + } + private: const bool *delimiter_; }; @@ -140,4 +148,4 @@ template <class Find, bool SkipEmpty = false> class TokenIter : public boost::it } // namespace util -#endif // UTIL_TOKENIZE_PIECE__ +#endif // UTIL_TOKENIZE_PIECE_H |