Merge remote-tracking branch 'upstream/master'

1 files changed, 17 insertions, 0 deletions

diff --git a/utils/stringlib.h b/utils/stringlib.h
index 2fdbfff8..aadec93c 100644
--- a/utils/stringlib.h
+++ b/utils/stringlib.h
@@ -16,6 +16,8 @@
 #include <sstream>
 #include <algorithm>
 
+#include "string_piece.hh"
+
 namespace {
 const char c_isspace[]=" \t\n\r\f\v"; // somewhat ridiculous, including characters nobody uses.
 const char common_isspace[]=" \t\n\r"; // even \n\r is borderline, but maybe you pass multiline DOS format text.
@@ -221,6 +223,21 @@ void VisitTokens(char *p,char *const end,F f) {
   }
 }
 
+inline std::vector<StringPiece> TokenizeMultisep(const StringPiece& text, const StringPiece& separator) {
+  std::vector<StringPiece> res;
+  size_t cur = 0;
+  while(cur < text.size()) {
+    const auto n = text.find(separator, cur);
+    if (n == StringPiece::npos) {
+      res.push_back(text.substr(cur));
+      break;
+    }
+    res.push_back(text.substr(cur, n - cur));
+    cur = n + separator.size();
+  }
+  return res;
+}
+
 template <class F>
 void VisitTokens(char *p,F f) {
   VisitTokens(p,p+std::strlen(p),f);