Tokenizer std::string str("w0rd, token-izer. pup's, U.S.a., us., hel.lo"); TermTokenizer tokenizer(str); std::vector<std::string> tokens(tokenizer.begin(), tokenizer.end());