summaryrefslogtreecommitdiff
path: root/klm/util/tokenize_piece.hh
diff options
context:
space:
mode:
authorChris Dyer <redpony@gmail.com>2014-10-13 00:42:37 -0400
committerChris Dyer <redpony@gmail.com>2014-10-13 00:42:37 -0400
commitb1ed81ef3216b212295afa76c5d20a56fb647204 (patch)
tree9633cdc1b8a341dfa58b0b7fec0e2cae44d28835 /klm/util/tokenize_piece.hh
parent1b17f61d359be6e1c3cea29f8c100db3bcdd73a0 (diff)
new kenlm
Diffstat (limited to 'klm/util/tokenize_piece.hh')
-rw-r--r--klm/util/tokenize_piece.hh16
1 files changed, 12 insertions, 4 deletions
diff --git a/klm/util/tokenize_piece.hh b/klm/util/tokenize_piece.hh
index 24eae8fb..908c8daf 100644
--- a/klm/util/tokenize_piece.hh
+++ b/klm/util/tokenize_piece.hh
@@ -1,5 +1,5 @@
-#ifndef UTIL_TOKENIZE_PIECE__
-#define UTIL_TOKENIZE_PIECE__
+#ifndef UTIL_TOKENIZE_PIECE_H
+#define UTIL_TOKENIZE_PIECE_H
#include "util/exception.hh"
#include "util/string_piece.hh"
@@ -7,7 +7,8 @@
#include <boost/iterator/iterator_facade.hpp>
#include <algorithm>
-#include <iostream>
+
+#include <string.h>
namespace util {
@@ -71,6 +72,13 @@ class BoolCharacter {
return StringPiece(in.data() + in.size(), 0);
}
+ template <unsigned Length> static void Build(const char (&characters)[Length], bool (&out)[256]) {
+ memset(out, 0, sizeof(out));
+ for (const char *i = characters; i != characters + Length; ++i) {
+ out[static_cast<unsigned char>(*i)] = true;
+ }
+ }
+
private:
const bool *delimiter_;
};
@@ -140,4 +148,4 @@ template <class Find, bool SkipEmpty = false> class TokenIter : public boost::it
} // namespace util
-#endif // UTIL_TOKENIZE_PIECE__
+#endif // UTIL_TOKENIZE_PIECE_H