1 : // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef BASE_STRING_TOKENIZER_H_
6 : #define BASE_STRING_TOKENIZER_H_
7 :
8 : #include <string>
9 :
10 : // StringTokenizerT is a simple string tokenizer class. It works like an
11 : // iterator that with each step (see the Advance method) updates members that
12 : // refer to the next token in the input string. The user may optionally
13 : // configure the tokenizer to return delimiters.
14 : //
15 : //
16 : // EXAMPLE 1:
17 : //
18 : // StringTokenizer t("this is a test", " ");
19 : // while (t.GetNext()) {
20 : // printf("%s\n", t.token().c_str());
21 : // }
22 : //
23 : // Output:
24 : //
25 : // this
26 : // is
27 : // a
28 : // test
29 : //
30 : //
31 : // EXAMPLE 2:
32 : //
33 : // StringTokenizer t("no-cache=\"foo, bar\", private", ", ");
34 : // t.set_quote_chars("\"");
35 : // while (t.GetNext()) {
36 : // printf("%s\n", t.token().c_str());
37 : // }
38 : //
39 : // Output:
40 : //
41 : // no-cache="foo, bar"
42 : // private
43 : //
44 : //
45 : // EXAMPLE 3:
46 : //
47 : // bool next_is_option = false, next_is_value = false;
48 : // std::string input = "text/html; charset=UTF-8; foo=bar";
49 : // StringTokenizer t(input, "; =");
50 : // t.set_options(StringTokenizer::RETURN_DELIMS);
51 : // while (t.GetNext()) {
52 : // if (t.token_is_delim()) {
53 : // switch (*t.token_begin()) {
54 : // case ';':
55 : // next_is_option = true;
56 : // break;
57 : // case '=':
58 : // next_is_value = true;
59 : // break;
60 : // }
61 : // } else {
62 : // const char* label;
63 : // if (next_is_option) {
64 : // label = "option-name";
65 : // next_is_option = false;
66 : // } else if (next_is_value) {
67 : // label = "option-value";
68 : // next_is_value = false;
69 : // } else {
70 : // label = "mime-type";
71 : // }
72 : // printf("%s: %s\n", label, t.token().c_str());
73 : // }
74 : // }
75 : //
76 : //
77 : template <class str, class const_iterator>
78 0 : class StringTokenizerT {
79 : public:
80 : typedef typename str::value_type char_type;
81 :
82 : // Options that may be pass to set_options()
83 : enum {
84 : // Specifies the delimiters should be returned as tokens
85 : RETURN_DELIMS = 1 << 0,
86 : };
87 :
88 0 : StringTokenizerT(const str& string,
89 0 : const str& delims) {
90 0 : Init(string.begin(), string.end(), delims);
91 0 : }
92 :
93 : StringTokenizerT(const_iterator string_begin,
94 : const_iterator string_end,
95 : const str& delims) {
96 : Init(string_begin, string_end, delims);
97 : }
98 :
99 : // Set the options for this tokenizer. By default, this is 0.
100 : void set_options(int options) { options_ = options; }
101 :
102 : // Set the characters to regard as quotes. By default, this is empty. When
103 : // a quote char is encountered, the tokenizer will switch into a mode where
104 : // it ignores delimiters that it finds. It switches out of this mode once it
105 : // finds another instance of the quote char. If a backslash is encountered
106 : // within a quoted string, then the next character is skipped.
107 : void set_quote_chars(const str& quotes) { quotes_ = quotes; }
108 :
109 : // Call this method to advance the tokenizer to the next delimiter. This
110 : // returns false if the tokenizer is complete. This method must be called
111 : // before calling any of the token* methods.
112 0 : bool GetNext() {
113 0 : AdvanceState state;
114 0 : token_is_delim_ = false;
115 0 : for (;;) {
116 0 : token_begin_ = token_end_;
117 0 : if (token_end_ == end_)
118 0 : return false;
119 0 : ++token_end_;
120 0 : if (AdvanceOne(&state, *token_begin_))
121 : break;
122 0 : if (options_ & RETURN_DELIMS) {
123 0 : token_is_delim_ = true;
124 0 : return true;
125 : }
126 : // else skip over delim
127 : }
128 0 : while (token_end_ != end_ && AdvanceOne(&state, *token_end_))
129 0 : ++token_end_;
130 0 : return true;
131 : }
132 :
133 : // Returns true if token is a delimiter. When the tokenizer is constructed
134 : // with the RETURN_DELIMS option, this method can be used to check if the
135 : // returned token is actually a delimiter.
136 : bool token_is_delim() const { return token_is_delim_; }
137 :
138 : // If GetNext() returned true, then these methods may be used to read the
139 : // value of the token.
140 : const_iterator token_begin() const { return token_begin_; }
141 : const_iterator token_end() const { return token_end_; }
142 0 : str token() const { return str(token_begin_, token_end_); }
143 :
144 : private:
145 0 : void Init(const_iterator string_begin,
146 : const_iterator string_end,
147 : const str& delims) {
148 0 : token_end_ = string_begin;
149 0 : end_ = string_end;
150 0 : delims_ = delims;
151 0 : options_ = 0;
152 0 : }
153 :
154 0 : bool IsDelim(char_type c) const {
155 0 : return delims_.find(c) != str::npos;
156 : }
157 :
158 0 : bool IsQuote(char_type c) const {
159 0 : return quotes_.find(c) != str::npos;
160 : }
161 :
162 : struct AdvanceState {
163 : bool in_quote;
164 : bool in_escape;
165 : char_type quote_char;
166 0 : AdvanceState() : in_quote(false), in_escape(false) {}
167 : };
168 :
169 : // Returns true if a delimiter was not hit.
170 0 : bool AdvanceOne(AdvanceState* state, char_type c) {
171 0 : if (state->in_quote) {
172 0 : if (state->in_escape) {
173 0 : state->in_escape = false;
174 0 : } else if (c == '\\') {
175 0 : state->in_escape = true;
176 0 : } else if (c == state->quote_char) {
177 0 : state->in_quote = false;
178 : }
179 : } else {
180 0 : if (IsDelim(c))
181 0 : return false;
182 0 : state->in_quote = IsQuote(state->quote_char = c);
183 : }
184 0 : return true;
185 : }
186 :
187 : const_iterator token_begin_;
188 : const_iterator token_end_;
189 : const_iterator end_;
190 : str delims_;
191 : str quotes_;
192 : int options_;
193 : bool token_is_delim_;
194 : };
195 :
196 : typedef StringTokenizerT<std::string, std::string::const_iterator>
197 : StringTokenizer;
198 : typedef StringTokenizerT<std::wstring, std::wstring::const_iterator>
199 : WStringTokenizer;
200 : typedef StringTokenizerT<std::string, const char*> CStringTokenizer;
201 :
202 : #endif // BASE_STRING_TOKENIZER_H_
|