Source-highlight Library
wordtokenizer.h
1 //
2 // Author: Lorenzo Bettini <http://www.lorenzobettini.it>, (C) 2004-2008
3 //
4 // Copyright: See COPYING file that comes with this distribution
5 //
6 
7 #ifndef WORDTOKENIZER_H_
8 #define WORDTOKENIZER_H_
9 
10 #include <string>
11 #include <algorithm>
12 #include <list>
13 
14 namespace srchilite {
15 
19 class WordTokenizer {
20 public:
26  typedef std::list<std::pair<std::string, std::string> > WordTokenizerResults;
27 
33  static void tokenize(const std::string &s, WordTokenizerResults &results);
34 };
35 
36 }
37 
38 #endif /*WORDTOKENIZER_H_*/
srchilite
C++ class: doctemplate.h.
Definition: bufferedoutput.cpp:13
srchilite::WordTokenizer
Tokenizes a paragraph separating words from spaces.
Definition: wordtokenizer.h:19
srchilite::WordTokenizer::WordTokenizerResults
std::list< std::pair< std::string, std::string > > WordTokenizerResults
Results of the tokenizer; each element is a pair where the first string represents a possible space a...
Definition: wordtokenizer.h:26
srchilite::WordTokenizer::tokenize
static void tokenize(const std::string &s, WordTokenizerResults &results)
Tokenizes the passed string and stores the results.
Definition: wordtokenizer.cpp:27