1 : /****************************************************************************\
2 : Copyright (c) 2002, NVIDIA Corporation.
3 :
4 : NVIDIA Corporation("NVIDIA") supplies this software to you in
5 : consideration of your agreement to the following terms, and your use,
6 : installation, modification or redistribution of this NVIDIA software
7 : constitutes acceptance of these terms. If you do not agree with these
8 : terms, please do not use, install, modify or redistribute this NVIDIA
9 : software.
10 :
11 : In consideration of your agreement to abide by the following terms, and
12 : subject to these terms, NVIDIA grants you a personal, non-exclusive
13 : license, under NVIDIA's copyrights in this original NVIDIA software (the
14 : "NVIDIA Software"), to use, reproduce, modify and redistribute the
15 : NVIDIA Software, with or without modifications, in source and/or binary
16 : forms; provided that if you redistribute the NVIDIA Software, you must
17 : retain the copyright notice of NVIDIA, this notice and the following
18 : text and disclaimers in all such redistributions of the NVIDIA Software.
19 : Neither the name, trademarks, service marks nor logos of NVIDIA
20 : Corporation may be used to endorse or promote products derived from the
21 : NVIDIA Software without specific prior written permission from NVIDIA.
22 : Except as expressly stated in this notice, no other rights or licenses
23 : express or implied, are granted by NVIDIA herein, including but not
24 : limited to any patent rights that may be infringed by your derivative
25 : works or by other works in which the NVIDIA Software may be
26 : incorporated. No hardware is licensed hereunder.
27 :
28 : THE NVIDIA SOFTWARE IS BEING PROVIDED ON AN "AS IS" BASIS, WITHOUT
29 : WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED,
30 : INCLUDING WITHOUT LIMITATION, WARRANTIES OR CONDITIONS OF TITLE,
31 : NON-INFRINGEMENT, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
32 : ITS USE AND OPERATION EITHER ALONE OR IN COMBINATION WITH OTHER
33 : PRODUCTS.
34 :
35 : IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT,
36 : INCIDENTAL, EXEMPLARY, CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
37 : TO, LOST PROFITS; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
38 : USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) OR ARISING IN ANY WAY
39 : OUT OF THE USE, REPRODUCTION, MODIFICATION AND/OR DISTRIBUTION OF THE
40 : NVIDIA SOFTWARE, HOWEVER CAUSED AND WHETHER UNDER THEORY OF CONTRACT,
41 : TORT (INCLUDING NEGLIGENCE), STRICT LIABILITY OR OTHERWISE, EVEN IF
42 : NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 : \****************************************************************************/
44 : //
45 : // tokens.c
46 : //
47 :
48 : #include <stdlib.h>
49 : #include <stdio.h>
50 : #include <string.h>
51 : #include <ctype.h>
52 :
53 : #include "compiler/compilerdebug.h"
54 : #include "compiler/preprocessor/slglobals.h"
55 : #include "compiler/util.h"
56 :
57 : ///////////////////////////////////////////////////////////////////////////////////////////////
58 : //////////////////////// Preprocessor and Token Recorder and Playback: ////////////////////////
59 : ///////////////////////////////////////////////////////////////////////////////////////////////
60 :
61 : /*
62 : * idstr()
63 : * Copy a string to a malloc'ed block and convert it into something suitable
64 : * for an ID
65 : *
66 : */
67 :
68 0 : static char *idstr(const char *fstr, MemoryPool *pool)
69 : {
70 : size_t len;
71 : char *str, *t;
72 : const char *f;
73 :
74 0 : len = strlen(fstr);
75 0 : if (!pool)
76 0 : str = (char *) malloc(len + 1);
77 : else
78 0 : str = (char *) mem_Alloc(pool, len + 1);
79 :
80 0 : for (f=fstr, t=str; *f; f++) {
81 0 : if (isalnum(*f)) *t++ = *f;
82 0 : else if (*f == '.' || *f == '/') *t++ = '_';
83 : }
84 0 : *t = 0;
85 0 : return str;
86 : } // idstr
87 :
88 :
89 : /*
90 : * lNewBlock()
91 : *
92 : */
93 :
94 0 : static TokenBlock *lNewBlock(TokenStream *fTok, MemoryPool *pool)
95 : {
96 : TokenBlock *lBlock;
97 :
98 0 : if (!pool)
99 0 : lBlock = (TokenBlock *) malloc(sizeof(TokenBlock) + 256);
100 : else
101 0 : lBlock = (TokenBlock *) mem_Alloc(pool, sizeof(TokenBlock) + 256);
102 0 : lBlock->count = 0;
103 0 : lBlock->current = 0;
104 0 : lBlock->data = (unsigned char *) lBlock + sizeof(TokenBlock);
105 0 : lBlock->max = 256;
106 0 : lBlock->next = NULL;
107 0 : if (fTok->head) {
108 0 : fTok->current->next = lBlock;
109 : } else {
110 0 : fTok->head = lBlock;
111 : }
112 0 : fTok->current = lBlock;
113 0 : return lBlock;
114 : } // lNewBlock
115 :
116 : /*
117 : * lAddByte()
118 : *
119 : */
120 :
121 0 : static void lAddByte(TokenStream *fTok, unsigned char fVal)
122 : {
123 : TokenBlock *lBlock;
124 0 : lBlock = fTok->current;
125 0 : if (lBlock->count >= lBlock->max)
126 0 : lBlock = lNewBlock(fTok, 0);
127 0 : lBlock->data[lBlock->count++] = fVal;
128 0 : } // lAddByte
129 :
130 :
131 :
132 : /*
133 : * lReadByte() - Get the next byte from a stream.
134 : *
135 : */
136 :
137 0 : static int lReadByte(TokenStream *pTok)
138 : {
139 : TokenBlock *lBlock;
140 0 : int lval = -1;
141 :
142 0 : lBlock = pTok->current;
143 0 : if (lBlock) {
144 0 : if (lBlock->current >= lBlock->count) {
145 0 : lBlock = lBlock->next;
146 0 : if (lBlock)
147 0 : lBlock->current = 0;
148 0 : pTok->current = lBlock;
149 : }
150 0 : if (lBlock)
151 0 : lval = lBlock->data[lBlock->current++];
152 : }
153 0 : return lval;
154 : } // lReadByte
155 :
156 : /////////////////////////////////////// Global Functions://////////////////////////////////////
157 :
158 : /*
159 : * NewTokenStream()
160 : *
161 : */
162 :
163 0 : TokenStream *NewTokenStream(const char *name, MemoryPool *pool)
164 : {
165 : TokenStream *pTok;
166 :
167 0 : if (!pool)
168 0 : pTok = (TokenStream *) malloc(sizeof(TokenStream));
169 : else
170 0 : pTok = (TokenStream*)mem_Alloc(pool, sizeof(TokenStream));
171 0 : pTok->next = NULL;
172 0 : pTok->name = idstr(name, pool);
173 0 : pTok->head = NULL;
174 0 : pTok->current = NULL;
175 0 : lNewBlock(pTok, pool);
176 0 : return pTok;
177 : } // NewTokenStream
178 :
179 : /*
180 : * DeleteTokenStream()
181 : *
182 : */
183 :
184 0 : void DeleteTokenStream(TokenStream *pTok)
185 : {
186 : TokenBlock *pBlock, *nBlock;
187 :
188 0 : if (pTok) {
189 0 : pBlock = pTok->head;
190 0 : while (pBlock) {
191 0 : nBlock = pBlock->next;
192 0 : free(pBlock);
193 0 : pBlock = nBlock;
194 : }
195 0 : if (pTok->name)
196 0 : free(pTok->name);
197 0 : free(pTok);
198 : }
199 0 : } // DeleteTokenStream
200 :
201 : /*
202 : * RecordToken() - Add a token to the end of a list for later playback or printout.
203 : *
204 : */
205 :
206 0 : void RecordToken(TokenStream *pTok, int token, yystypepp * yylvalpp)
207 : {
208 : const char *s;
209 0 : char *str=NULL;
210 :
211 0 : if (token > 256)
212 0 : lAddByte(pTok, (unsigned char)((token & 0x7f) + 0x80));
213 : else
214 0 : lAddByte(pTok, (unsigned char)(token & 0x7f));
215 0 : switch (token) {
216 : case CPP_IDENTIFIER:
217 : case CPP_TYPEIDENTIFIER:
218 : case CPP_STRCONSTANT:
219 0 : s = GetAtomString(atable, yylvalpp->sc_ident);
220 0 : while (*s)
221 0 : lAddByte(pTok, (unsigned char) *s++);
222 0 : lAddByte(pTok, 0);
223 0 : break;
224 : case CPP_FLOATCONSTANT:
225 : case CPP_INTCONSTANT:
226 0 : str=yylvalpp->symbol_name;
227 0 : while (*str){
228 0 : lAddByte(pTok, (unsigned char) *str++);
229 : }
230 0 : lAddByte(pTok, 0);
231 0 : break;
232 : case '(':
233 0 : lAddByte(pTok, (unsigned char)(yylvalpp->sc_int ? 1 : 0));
234 : default:
235 0 : break;
236 : }
237 0 : } // RecordToken
238 :
239 : /*
240 : * RewindTokenStream() - Reset a token stream in preperation for reading.
241 : *
242 : */
243 :
244 0 : void RewindTokenStream(TokenStream *pTok)
245 : {
246 0 : if (pTok->head) {
247 0 : pTok->current = pTok->head;
248 0 : pTok->current->current = 0;
249 : }
250 0 : } // RewindTokenStream
251 :
252 : /*
253 : * ReadToken() - Read the next token from a stream.
254 : *
255 : */
256 :
257 0 : int ReadToken(TokenStream *pTok, yystypepp * yylvalpp)
258 : {
259 : char symbol_name[MAX_SYMBOL_NAME_LEN + 1];
260 : char string_val[MAX_STRING_LEN + 1];
261 : int ltoken, len;
262 : char ch;
263 :
264 0 : ltoken = lReadByte(pTok);
265 0 : if (ltoken >= 0) {
266 0 : if (ltoken > 127)
267 0 : ltoken += 128;
268 0 : switch (ltoken) {
269 : case CPP_IDENTIFIER:
270 : case CPP_TYPEIDENTIFIER:
271 0 : len = 0;
272 0 : ch = lReadByte(pTok);
273 0 : while ((ch >= 'a' && ch <= 'z') ||
274 0 : (ch >= 'A' && ch <= 'Z') ||
275 0 : (ch >= '0' && ch <= '9') ||
276 : ch == '_')
277 : {
278 0 : if (len < MAX_SYMBOL_NAME_LEN) {
279 0 : symbol_name[len++] = ch;
280 0 : ch = lReadByte(pTok);
281 : }
282 : }
283 0 : symbol_name[len] = '\0';
284 0 : assert(ch == '\0');
285 0 : yylvalpp->sc_ident = LookUpAddString(atable, symbol_name);
286 0 : return CPP_IDENTIFIER;
287 : break;
288 : case CPP_STRCONSTANT:
289 0 : len = 0;
290 0 : while ((ch = lReadByte(pTok)) != 0)
291 0 : if (len < MAX_STRING_LEN)
292 0 : string_val[len++] = ch;
293 0 : string_val[len] = '\0';
294 0 : yylvalpp->sc_ident = LookUpAddString(atable, string_val);
295 0 : break;
296 : case CPP_FLOATCONSTANT:
297 0 : len = 0;
298 0 : ch = lReadByte(pTok);
299 0 : while ((ch >= '0' && ch <= '9')||(ch=='e'||ch=='E'||ch=='.')||(ch=='+'||ch=='-'))
300 : {
301 0 : if (len < MAX_SYMBOL_NAME_LEN) {
302 0 : symbol_name[len++] = ch;
303 0 : ch = lReadByte(pTok);
304 : }
305 : }
306 0 : symbol_name[len] = '\0';
307 0 : assert(ch == '\0');
308 0 : strcpy(yylvalpp->symbol_name,symbol_name);
309 0 : yylvalpp->sc_fval=(float)atof_dot(yylvalpp->symbol_name);
310 0 : break;
311 : case CPP_INTCONSTANT:
312 0 : len = 0;
313 0 : ch = lReadByte(pTok);
314 0 : while ((ch >= '0' && ch <= '9'))
315 : {
316 0 : if (len < MAX_SYMBOL_NAME_LEN) {
317 0 : symbol_name[len++] = ch;
318 0 : ch = lReadByte(pTok);
319 : }
320 : }
321 0 : symbol_name[len] = '\0';
322 0 : assert(ch == '\0');
323 0 : strcpy(yylvalpp->symbol_name,symbol_name);
324 0 : yylvalpp->sc_int=atoi(yylvalpp->symbol_name);
325 0 : break;
326 : case '(':
327 0 : yylvalpp->sc_int = lReadByte(pTok);
328 0 : break;
329 : }
330 0 : return ltoken;
331 : }
332 0 : return EOF_SY;
333 : } // ReadToken
334 :
335 : typedef struct TokenInputSrc {
336 : InputSrc base;
337 : TokenStream *tokens;
338 : int (*final)(CPPStruct *);
339 : } TokenInputSrc;
340 :
341 0 : static int scan_token(TokenInputSrc *in, yystypepp * yylvalpp)
342 : {
343 0 : int token = ReadToken(in->tokens, yylvalpp);
344 : int (*final)(CPPStruct *);
345 0 : cpp->tokenLoc->file = cpp->currentInput->name;
346 0 : cpp->tokenLoc->line = cpp->currentInput->line;
347 0 : if (token == '\n') {
348 0 : in->base.line++;
349 0 : return token;
350 : }
351 0 : if (token > 0) return token;
352 0 : cpp->currentInput = in->base.prev;
353 0 : final = in->final;
354 0 : free(in);
355 0 : if (final && !final(cpp)) return -1;
356 0 : return cpp->currentInput->scan(cpp->currentInput, yylvalpp);
357 : }
358 :
359 0 : int ReadFromTokenStream(TokenStream *ts, int name, int (*final)(CPPStruct *))
360 : {
361 0 : TokenInputSrc *in = malloc(sizeof(TokenInputSrc));
362 0 : memset(in, 0, sizeof(TokenInputSrc));
363 0 : in->base.name = name;
364 0 : in->base.prev = cpp->currentInput;
365 0 : in->base.scan = (int (*)(InputSrc *, yystypepp *))scan_token;
366 0 : in->base.line = 1;
367 0 : in->tokens = ts;
368 0 : in->final = final;
369 0 : RewindTokenStream(ts);
370 0 : cpp->currentInput = &in->base;
371 0 : return 1;
372 : }
373 :
374 : typedef struct UngotToken {
375 : InputSrc base;
376 : int token;
377 : yystypepp lval;
378 : } UngotToken;
379 :
380 0 : static int reget_token(UngotToken *t, yystypepp * yylvalpp)
381 : {
382 0 : int token = t->token;
383 0 : *yylvalpp = t->lval;
384 0 : cpp->currentInput = t->base.prev;
385 0 : free(t);
386 0 : return token;
387 : }
388 :
389 0 : void UngetToken(int token, yystypepp * yylvalpp) {
390 0 : UngotToken *t = malloc(sizeof(UngotToken));
391 0 : memset(t, 0, sizeof(UngotToken));
392 0 : t->token = token;
393 0 : t->lval = *yylvalpp;
394 0 : t->base.scan = (void *)reget_token;
395 0 : t->base.prev = cpp->currentInput;
396 0 : t->base.name = cpp->currentInput->name;
397 0 : t->base.line = cpp->currentInput->line;
398 0 : cpp->currentInput = &t->base;
399 0 : }
400 :
401 :
402 0 : void DumpTokenStream(FILE *fp, TokenStream *s, yystypepp * yylvalpp) {
403 : int token;
404 : char str[100];
405 :
406 0 : if (fp == 0) fp = stdout;
407 0 : RewindTokenStream(s);
408 0 : while ((token = ReadToken(s, yylvalpp)) > 0) {
409 0 : switch (token) {
410 : case CPP_IDENTIFIER:
411 : case CPP_TYPEIDENTIFIER:
412 0 : sprintf(str, "%s ", GetAtomString(atable, yylvalpp->sc_ident));
413 0 : break;
414 : case CPP_STRCONSTANT:
415 0 : sprintf(str, "\"%s\"", GetAtomString(atable, yylvalpp->sc_ident));
416 0 : break;
417 : case CPP_FLOATCONSTANT:
418 : //printf("%g9.6 ", yylvalpp->sc_fval);
419 0 : break;
420 : case CPP_INTCONSTANT:
421 : //printf("%d ", yylvalpp->sc_int);
422 0 : break;
423 : default:
424 0 : if (token >= 127)
425 0 : sprintf(str, "%s ", GetAtomString(atable, token));
426 : else
427 0 : sprintf(str, "%c", token);
428 0 : break;
429 : }
430 0 : CPPDebugLogMsg(str);
431 : }
432 0 : }
433 :
434 : ///////////////////////////////////////////////////////////////////////////////////////////////
435 : /////////////////////////////////////// End of tokens.c ///////////////////////////////////////
436 : ///////////////////////////////////////////////////////////////////////////////////////////////
|