diff options
| author | Patrick Simianer <p@simianer.de> | 2014-10-13 19:03:48 +0100 |
|---|---|---|
| committer | Patrick Simianer <p@simianer.de> | 2014-10-13 19:03:48 +0100 |
| commit | cb9fb7088dde35881516c088db402abe747d49fa (patch) | |
| tree | a91e4935a7941f1b261f76d88ab41fa3078a1891 /klm/util/tokenize_piece.hh | |
| parent | 0a00e57e921c8eca8e02364db7d2e6607bfdcebc (diff) | |
| parent | b1ed81ef3216b212295afa76c5d20a56fb647204 (diff) | |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'klm/util/tokenize_piece.hh')
| -rw-r--r-- | klm/util/tokenize_piece.hh | 16 |
1 files changed, 12 insertions, 4 deletions
diff --git a/klm/util/tokenize_piece.hh b/klm/util/tokenize_piece.hh index 24eae8fb..908c8daf 100644 --- a/klm/util/tokenize_piece.hh +++ b/klm/util/tokenize_piece.hh @@ -1,5 +1,5 @@ -#ifndef UTIL_TOKENIZE_PIECE__ -#define UTIL_TOKENIZE_PIECE__ +#ifndef UTIL_TOKENIZE_PIECE_H +#define UTIL_TOKENIZE_PIECE_H #include "util/exception.hh" #include "util/string_piece.hh" @@ -7,7 +7,8 @@ #include <boost/iterator/iterator_facade.hpp> #include <algorithm> -#include <iostream> + +#include <string.h> namespace util { @@ -71,6 +72,13 @@ class BoolCharacter { return StringPiece(in.data() + in.size(), 0); } + template <unsigned Length> static void Build(const char (&characters)[Length], bool (&out)[256]) { + memset(out, 0, sizeof(out)); + for (const char *i = characters; i != characters + Length; ++i) { + out[static_cast<unsigned char>(*i)] = true; + } + } + private: const bool *delimiter_; }; @@ -140,4 +148,4 @@ template <class Find, bool SkipEmpty = false> class TokenIter : public boost::it } // namespace util -#endif // UTIL_TOKENIZE_PIECE__ +#endif // UTIL_TOKENIZE_PIECE_H |