| author | David Anderson <dvander@alliedmods.net> |
| Sun Jul 01 14:52:57 2012 -0700 (2012-07-01) | |
| changeset 21 | af5d8e985502 |
| parent 10 | 0c3bcd50e361 |
| child 22 | 765ab1b672a3 |
| permissions | -rw-r--r-- |
1 /*
2 * Copyright (C) 2012 David Anderson
3 *
4 * This file is part of JITCraft.
5 *
6 * JITCraft is free software: you can redistribute it and/or modify it under
7 * the terms of the GNU General Public License as published by the Free
8 * Software Foundation, either version 3 of the License, or (at your option)
9 * any later version.
10 *
11 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * JITCraft. If not, see http://www.gnu.org/licenses/.
17 */
18 #include <assert.h>
19 #include <ctype.h>
20 #include "Scanner.h"
22 using namespace jc;
24 const char *jc::TokenNames[] =
25 {
26 #define _(name, str) str,
27 TOKENMAP(_)
28 #undef _
29 NULL
30 };
32 Scanner::Scanner(const char *stream, size_t length)
33 : stream_(stream),
34 length_(length),
35 pushed_(false),
36 saved_('\0'),
37 pos_(stream_)
38 {
39 }
41 char
42 Scanner::getChar()
43 {
44 if (saved_ != '\0') {
45 char temp = saved_;
46 saved_ = '\0';
47 return temp;
48 }
49 if (pos_ >= stream_ + length_)
50 return '\0';
51 return *pos_++;
52 }
54 void
55 Scanner::putChar(char c)
56 {
57 assert(saved_ == '\0');
58 saved_ = c;
59 }
61 void
62 Scanner::eatSingleLine()
63 {
64 char c;
65 while ((c = getChar()) != '\0') {
66 if (c == '\n' || c == '\r')
67 break;
68 }
69 }
71 bool
72 Scanner::eatComment()
73 {
74 char c;
76 while ((c = getChar()) != '\0') {
77 while (c == '*') {
78 c = getChar();
79 if (c == '\0')
80 return false;
81 if (c == '/')
82 return true;
83 }
84 }
86 return false;
87 }
89 static inline bool
90 IsDigit(char c)
91 {
92 return c >= '0' && c <= '9';
93 }
95 // Based off the logic in sc2.c's ftoi()...
96 double
97 Scanner::ParseDouble(const char *string)
98 {
99 const char *ptr = string;
101 double number = 0.0;
102 while (IsDigit(*ptr)) {
103 number = (number * 10) + (*ptr - '0');
104 ptr++;
105 }
107 assert(*ptr == '.');
108 ptr++;
110 double fraction = 0.0;
111 double multiplier = 1.0;
112 while (IsDigit(*ptr)) {
113 fraction = (fraction * 10) + (*ptr - '0');
114 multiplier = multiplier / 10.0;
115 ptr++;
116 }
118 number += fraction * multiplier;
120 if (*ptr++ == 'e') {
121 int sign = 1;
122 if (*ptr == '-') {
123 sign = -1;
124 ptr++;
125 }
127 int exponent = 0;
128 while (IsDigit(*ptr)) {
129 exponent = (exponent * 10) + (*ptr - '0');
130 ptr++;
131 }
133 multiplier = pow(10.0, exponent * sign);
134 number *= multiplier;
135 }
137 return number;
138 }
140 static inline bool
141 IsHexDigit(char c)
142 {
143 return IsDigit(c) ||
144 (c >= 'a' && c <= 'f') ||
145 (c >= 'A' && c <= 'F');
146 }
148 // Based on the logic for litchar() in sc2.c.
149 int
150 Scanner::readEscapeCode()
151 {
152 char c = getChar();
153 if (c == '\\')
154 return c;
156 switch (c) {
157 case 'a':
158 return '\a';
159 case 'b':
160 return '\b';
161 case 'e':
162 return 27; // Apparently \e is non-standard.
163 case 'f':
164 return '\f';
165 case 'n':
166 return '\n';
167 case 'r':
168 return '\r';
169 case 't':
170 return '\t';
171 case '\v':
172 return '\v';
174 case 'x':
175 {
176 unsigned digits = 0;
177 char r = 0;
179 c = getChar();
180 while (IsHexDigit(c) && digits < 2) {
181 if (IsDigit(c))
182 c = (c << 4) + (c - '0');
183 else
184 c = (c << 4) + (tolower(c) - 'a' + 10);
185 digits++;
186 c = getChar();
187 }
189 // Swallow a trailing ';'
190 if (c != ';')
191 putChar(c);
193 return r;
194 }
196 case '\'':
197 case '\"':
198 case '%':
199 return c;
201 default:
202 {
203 if (IsDigit(c)) {
204 // \ddd
205 char r = 0;
206 while (IsDigit(c)) {
207 r = r * 10 + (c - '0');
208 c = getChar();
209 }
211 // Swallow a trailing ;
212 if (c != ';')
213 putChar(c);
215 return r;
216 }
217 assert(false);
218 return INT_MAX;
219 }
220 }
221 }
223 TokenKind
224 Scanner::stringLiteral()
225 {
226 literal_.clear();
228 for (;;) {
229 char c = getChar();
230 if (c == '\"')
231 break;
232 if (c == '\r' || c == '\n' || c == '\0') {
233 assert(false);
234 return TOK_ERROR;
235 }
236 if (c == '\\') {
237 int code = readEscapeCode();
238 if (code == INT_MAX)
239 return TOK_ERROR;
240 c = char(code);
241 }
242 literal_ += c;
243 }
245 return TOK_STRING_LITERAL;
246 }
248 TokenKind
249 Scanner::numberLiteral(char first)
250 {
251 literal_ = first;
253 char c;
254 for (;;) {
255 c = getChar();
256 if (!IsDigit(c))
257 break;
258 literal_ += c;
259 }
261 if (c != '.') {
262 putChar(c);
263 return TOK_INTEGER_LITERAL;
264 }
265 literal_ += c;
267 c = getChar();
268 if (!IsDigit(c)) {
269 assert(false);
270 return TOK_ERROR;
271 }
272 literal_ += c;
274 for (;;) {
275 c = getChar();
276 if (!IsDigit(c))
277 break;
278 literal_ += c;
279 }
281 if (c != 'e') {
282 putChar(c);
283 } else {
284 literal_ += c;
285 c = getChar();
286 if (c == '-') {
287 literal_ += c;
288 c = getChar();
289 }
290 if (!IsDigit(c)) {
291 assert(false);
292 return TOK_ERROR;
293 }
294 literal_ += c;
295 for (;;) {
296 c = getChar();
297 if (!IsDigit(c)) {
298 putChar(c);
299 break;
300 }
301 literal_ += c;
302 }
303 }
305 return TOK_FLOAT_LITERAL;
306 }
308 TokenKind
309 Scanner::identifier(char first)
310 {
311 literal_ = first;
313 char c;
314 for (;;) {
315 c = getChar();
316 if (!isascii(c) || (!isalnum(c) && c != '_')) {
317 if (c != ':')
318 putChar(c);
319 break;
320 }
321 literal_ += c;
322 }
324 if (literal_ == "new")
325 return TOK_NEW;
326 if (literal_ == "for")
327 return TOK_FOR;
328 if (literal_ == "return")
329 return TOK_RETURN;
330 if (literal_ == "if")
331 return TOK_IF;
332 if (literal_ == "else")
333 return TOK_ELSE;
334 if (literal_ == "enum")
335 return TOK_ENUM;
336 if (literal_ == "while")
337 return TOK_WHILE;
338 if (literal_ == "do")
339 return TOK_DO;
340 if (literal_ == "break")
341 return TOK_BREAK;
342 if (literal_ == "continue")
343 return TOK_CONTINUE;
344 return (c == ':') ? TOK_LABEL : TOK_NAME;
345 }
347 TokenKind
348 Scanner::next()
349 {
350 assert(!pushed_);
352 for (;;) {
353 char c = getChar();
354 switch (c) {
355 case '/':
356 {
357 c = getChar();
358 if (c == '/') {
359 eatSingleLine();
360 break;
361 }
362 if (c == '*') {
363 if (!eatComment())
364 return TOK_ERROR;
365 break;
366 }
367 putChar(c);
368 return TOK_SLASH;
369 }
371 case ';': return TOK_SEMICOLON;
372 case '{': return TOK_LBRACE;
373 case '}': return TOK_RBRACE;
374 case '(': return TOK_LPAREN;
375 case ')': return TOK_RPAREN;
376 case '[': return TOK_LBRACKET;
377 case ']': return TOK_RBRACKET;
379 case '*':
380 c = getChar();
381 if (c == '=')
382 return TOK_ASSIGN_MUL;
383 putChar(c);
384 return TOK_STAR;
386 case '+':
387 c = getChar();
388 if (c == '=')
389 return TOK_ASSIGN_ADD;
390 putChar(c);
391 return TOK_PLUS;
393 case '&':
394 return TOK_BITAND;
396 case '|':
397 return TOK_BITOR;
399 case '^':
400 return TOK_BITXOR;
402 case '-': return TOK_MINUS;
403 case ',': return TOK_COMMA;
404 case '\0': return TOK_EOF;
406 case '=':
407 c = getChar();
408 if (c == '=')
409 return TOK_EQUALS;
410 putChar(c);
411 return TOK_ASSIGN;
413 case '<':
414 c = getChar();
415 if (c == '=')
416 return TOK_LE;
417 if (c == '<') {
418 c = getChar();
419 if (c == '=')
420 return TOK_ASSIGN_SHL;
421 return TOK_SHL;
422 }
423 putChar(c);
424 return TOK_LT;
426 case '>':
427 c = getChar();
428 if (c == '=')
429 return TOK_GE;
430 if (c == '>') {
431 c = getChar();
432 if (c == '>')
433 return TOK_USHR;
434 return TOK_SHR;
435 }
436 putChar(c);
437 return TOK_GT;
439 case ' ':
440 case '\t':
441 case '\r':
442 case '\f':
443 case '\n':
444 break;
446 case '"':
447 return stringLiteral();
449 default:
450 if (c >= '0' && c <= '9')
451 return numberLiteral(c);
452 if (c == '_' || (isascii(c) && isalpha(c)))
453 return identifier(c);
454 assert(false);
455 return TOK_ERROR;
456 }
457 }
458 }
460 TokenKind
461 Scanner::scan()
462 {
463 if (pushed_) {
464 pushed_ = false;
465 return lastToken_;
466 }
467 lastToken_ = next();
468 return lastToken_;
469 }
471 TokenKind
472 Scanner::peek()
473 {
474 if (pushed_)
475 return lastToken_;
476 scan();
477 pushed_ = true;
478 return lastToken_;
479 }