diff options
| author | Chris Dyer <redpony@gmail.com> | 2014-10-13 00:42:37 -0400 |
|---|---|---|
| committer | Chris Dyer <redpony@gmail.com> | 2014-10-13 00:42:37 -0400 |
| commit | b1ed81ef3216b212295afa76c5d20a56fb647204 (patch) | |
| tree | 9633cdc1b8a341dfa58b0b7fec0e2cae44d28835 /klm/util/tokenize_piece.hh | |
| parent | 1b17f61d359be6e1c3cea29f8c100db3bcdd73a0 (diff) | |
new kenlm
Diffstat (limited to 'klm/util/tokenize_piece.hh')
| -rw-r--r-- | klm/util/tokenize_piece.hh | 16 |
1 files changed, 12 insertions, 4 deletions
diff --git a/klm/util/tokenize_piece.hh b/klm/util/tokenize_piece.hh index 24eae8fb..908c8daf 100644 --- a/klm/util/tokenize_piece.hh +++ b/klm/util/tokenize_piece.hh @@ -1,5 +1,5 @@ -#ifndef UTIL_TOKENIZE_PIECE__ -#define UTIL_TOKENIZE_PIECE__ +#ifndef UTIL_TOKENIZE_PIECE_H +#define UTIL_TOKENIZE_PIECE_H #include "util/exception.hh" #include "util/string_piece.hh" @@ -7,7 +7,8 @@ #include <boost/iterator/iterator_facade.hpp> #include <algorithm> -#include <iostream> + +#include <string.h> namespace util { @@ -71,6 +72,13 @@ class BoolCharacter { return StringPiece(in.data() + in.size(), 0); } + template <unsigned Length> static void Build(const char (&characters)[Length], bool (&out)[256]) { + memset(out, 0, sizeof(out)); + for (const char *i = characters; i != characters + Length; ++i) { + out[static_cast<unsigned char>(*i)] = true; + } + } + private: const bool *delimiter_; }; @@ -140,4 +148,4 @@ template <class Find, bool SkipEmpty = false> class TokenIter : public boost::it } // namespace util -#endif // UTIL_TOKENIZE_PIECE__ +#endif // UTIL_TOKENIZE_PIECE_H |