| author | David Anderson <dvander@alliedmods.net> |
| Sun Jul 08 15:23:48 2012 -0700 (2012-07-08) | |
| changeset 62 | e04c5be84792 |
| parent 59 | b7d61ddb3c8d |
| child 63 | dd57743d220e |
| permissions | -rw-r--r-- |
1 /*
2 * Copyright (C) 2012 David Anderson
3 *
4 * This file is part of JITCraft.
5 *
6 * JITCraft is free software: you can redistribute it and/or modify it under
7 * the terms of the GNU General Public License as published by the Free
8 * Software Foundation, either version 3 of the License, or (at your option)
9 * any later version.
10 *
11 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * JITCraft. If not, see http://www.gnu.org/licenses/.
17 */
18 #include <assert.h>
19 #include <ctype.h>
20 #include <math.h>
21 #include "Scanner.h"
22 #include "Utility.h"
24 using namespace ke;
26 const char *ke::TokenNames[] =
27 {
28 #define _(name, str) str,
29 TOKENMAP(_)
30 #undef _
31 NULL
32 };
34 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
35 : cc_(cc),
36 stream_(stream),
37 length_(length),
38 pushed_(false),
39 saved_('\0'),
40 pos_(stream_)
41 {
42 cur_.line = 1;
43 cur_.col = 0;
44 last_ = cur_;
45 }
47 char
48 Scanner::getChar()
49 {
50 if (saved_ != '\0') {
51 char temp = saved_;
52 saved_ = '\0';
53 return temp;
54 }
55 if (pos_ >= stream_ + length_)
56 return '\0';
57 last_.col = cur_.col++;
58 return *pos_++;
59 }
61 void
62 Scanner::putChar(char c)
63 {
64 assert(saved_ == '\0');
65 saved_ = c;
66 cur_.col = last_.col;
67 }
69 void
70 Scanner::calcNewLine(char c)
71 {
72 cur_.line++;
73 cur_.col = 0;
74 last_ = cur_;
76 char o;
77 if (c == '\r' && ((o = getChar()) != '\n'))
78 putChar(o);
79 }
81 void
82 Scanner::eatSingleLine()
83 {
84 char c;
85 while ((c = getChar()) != '\0') {
86 if (c == '\n' || c == '\r') {
87 calcNewLine(c);
88 break;
89 }
90 }
91 }
93 bool
94 Scanner::eatComment()
95 {
96 char c;
98 while ((c = getChar()) != '\0') {
99 while (c == '*') {
100 c = getChar();
101 if (c == '\0')
102 return false;
103 if (c == '/')
104 return true;
105 }
106 if (c == '\r' || c == '\n')
107 calcNewLine(c);
108 }
110 return false;
111 }
113 static inline bool
114 IsDigit(char c)
115 {
116 return c >= '0' && c <= '9';
117 }
119 // Based off the logic in sc2.c's ftoi()...
120 double
121 Scanner::ParseDouble(const char *string)
122 {
123 const char *ptr = string;
125 double number = 0.0;
126 while (IsDigit(*ptr)) {
127 number = (number * 10) + (*ptr - '0');
128 ptr++;
129 }
131 assert(*ptr == '.');
132 ptr++;
134 double fraction = 0.0;
135 double multiplier = 1.0;
136 while (IsDigit(*ptr)) {
137 fraction = (fraction * 10) + (*ptr - '0');
138 multiplier = multiplier / 10.0;
139 ptr++;
140 }
142 number += fraction * multiplier;
144 if (*ptr++ == 'e') {
145 int sign = 1;
146 if (*ptr == '-') {
147 sign = -1;
148 ptr++;
149 }
151 int exponent = 0;
152 while (IsDigit(*ptr)) {
153 exponent = (exponent * 10) + (*ptr - '0');
154 ptr++;
155 }
157 multiplier = pow(10.0, exponent * sign);
158 number *= multiplier;
159 }
161 return number;
162 }
164 static inline bool
165 IsHexDigit(char c)
166 {
167 return IsDigit(c) ||
168 (c >= 'a' && c <= 'f') ||
169 (c >= 'A' && c <= 'F');
170 }
172 // Based on the logic for litchar() in sc2.c.
173 int
174 Scanner::readEscapeCode()
175 {
176 char c = getChar();
177 if (c == '\\')
178 return c;
180 switch (c) {
181 case 'a':
182 return '\a';
183 case 'b':
184 return '\b';
185 case 'e':
186 return 27; // Apparently \e is non-standard.
187 case 'f':
188 return '\f';
189 case 'n':
190 return '\n';
191 case 'r':
192 return '\r';
193 case 't':
194 return '\t';
195 case '\v':
196 return '\v';
198 case 'x':
199 {
200 unsigned digits = 0;
201 char r = 0;
203 c = getChar();
204 while (IsHexDigit(c) && digits < 2) {
205 if (IsDigit(c))
206 c = (c << 4) + (c - '0');
207 else
208 c = (c << 4) + (tolower(c) - 'a' + 10);
209 digits++;
210 c = getChar();
211 }
213 // Swallow a trailing ';'
214 if (c != ';')
215 putChar(c);
217 return r;
218 }
220 case '\'':
221 case '\"':
222 case '%':
223 return c;
225 default:
226 {
227 if (IsDigit(c)) {
228 // \ddd
229 char r = 0;
230 while (IsDigit(c)) {
231 r = r * 10 + (c - '0');
232 c = getChar();
233 }
235 // Swallow a trailing ;
236 if (c != ';')
237 putChar(c);
239 return r;
240 }
241 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
242 return INT_MAX;
243 }
244 }
245 }
247 TokenKind
248 Scanner::stringLiteral()
249 {
250 literal_.clear();
252 for (;;) {
253 char c = getChar();
254 if (c == '\"')
255 break;
256 if (c == '\r' || c == '\n' || c == '\0') {
257 cc_.reportError(begin(), Message_UnterminatedString);
258 return TOK_ERROR;
259 }
260 if (c == '\\') {
261 int code = readEscapeCode();
262 if (code == INT_MAX)
263 return TOK_ERROR;
264 c = char(code);
265 }
266 literal_ += c;
267 }
269 return TOK_STRING_LITERAL;
270 }
272 TokenKind
273 Scanner::numberLiteral(char first)
274 {
275 literal_ = first;
277 char c;
278 for (;;) {
279 c = getChar();
280 if (!IsDigit(c))
281 break;
282 literal_ += c;
283 }
285 if (c != '.') {
286 putChar(c);
287 return TOK_INTEGER_LITERAL;
288 }
289 literal_ += c;
291 c = getChar();
292 if (!IsDigit(c)) {
293 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
294 return TOK_ERROR;
295 }
296 literal_ += c;
298 for (;;) {
299 c = getChar();
300 if (!IsDigit(c))
301 break;
302 literal_ += c;
303 }
305 if (c != 'e') {
306 putChar(c);
307 } else {
308 literal_ += c;
309 c = getChar();
310 if (c == '-') {
311 literal_ += c;
312 c = getChar();
313 }
314 if (!IsDigit(c)) {
315 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
316 return TOK_ERROR;
317 }
318 literal_ += c;
319 for (;;) {
320 c = getChar();
321 if (!IsDigit(c)) {
322 putChar(c);
323 break;
324 }
325 literal_ += c;
326 }
327 }
329 return TOK_FLOAT_LITERAL;
330 }
332 TokenKind
333 Scanner::identifier(char first)
334 {
335 literal_ = first;
337 char c;
338 for (;;) {
339 c = getChar();
340 if (!isascii(c) || (!isalnum(c) && c != '_')) {
341 if (c != ':')
342 putChar(c);
343 break;
344 }
345 literal_ += c;
346 }
348 if (literal_ == "new")
349 return TOK_NEW;
350 if (literal_ == "for")
351 return TOK_FOR;
352 if (literal_ == "return")
353 return TOK_RETURN;
354 if (literal_ == "if")
355 return TOK_IF;
356 if (literal_ == "else")
357 return TOK_ELSE;
358 if (literal_ == "enum")
359 return TOK_ENUM;
360 if (literal_ == "while")
361 return TOK_WHILE;
362 if (literal_ == "do")
363 return TOK_DO;
364 if (literal_ == "break")
365 return TOK_BREAK;
366 if (literal_ == "continue")
367 return TOK_CONTINUE;
368 return (c == ':') ? TOK_LABEL : TOK_NAME;
369 }
371 TokenKind
372 Scanner::next()
373 {
374 assert(!pushed_);
376 for (;;) {
377 begin_ = cur_;
379 char c = getChar();
380 switch (c) {
381 case '/':
382 {
383 c = getChar();
384 if (c == '/') {
385 eatSingleLine();
386 break;
387 }
388 if (c == '*') {
389 if (!eatComment()) {
390 cc_.reportError(begin(), Message_UnterminatedComment);
391 return TOK_ERROR;
392 }
393 break;
394 }
395 putChar(c);
396 return TOK_SLASH;
397 }
399 case '\n':
400 case '\r':
401 calcNewLine(c);
402 break;
404 case ';': return TOK_SEMICOLON;
405 case '{': return TOK_LBRACE;
406 case '}': return TOK_RBRACE;
407 case '(': return TOK_LPAREN;
408 case ')': return TOK_RPAREN;
409 case '[': return TOK_LBRACKET;
410 case ']': return TOK_RBRACKET;
412 case '*':
413 c = getChar();
414 if (c == '=')
415 return TOK_ASSIGN_MUL;
416 putChar(c);
417 return TOK_STAR;
419 case '+':
420 c = getChar();
421 if (c == '=')
422 return TOK_ASSIGN_ADD;
423 if (c == '+')
424 return TOK_INCREMENT;
425 putChar(c);
426 return TOK_PLUS;
428 case '&':
429 c = getChar();
430 if (c == '=')
431 return TOK_ASSIGN_BITAND;
432 putChar(c);
433 return TOK_BITAND;
435 case '|':
436 c = getChar();
437 if (c == '=')
438 return TOK_ASSIGN_BITOR;
439 putChar(c);
440 return TOK_BITOR;
442 case '^':
443 c = getChar();
444 if (c == '=')
445 return TOK_ASSIGN_BITXOR;
446 putChar(c);
447 return TOK_BITXOR;
449 case '-':
450 c = getChar();
451 if (c == '=')
452 return TOK_ASSIGN_SUB;
453 if (c == '-')
454 return TOK_DECREMENT;
455 putChar(c);
456 return TOK_MINUS;
458 case ',': return TOK_COMMA;
459 case '\0': return TOK_EOF;
461 case '=':
462 c = getChar();
463 if (c == '=')
464 return TOK_EQUALS;
465 putChar(c);
466 return TOK_ASSIGN;
468 case '<':
469 c = getChar();
470 if (c == '=')
471 return TOK_LE;
472 if (c == '<') {
473 c = getChar();
474 if (c == '=')
475 return TOK_ASSIGN_SHL;
476 return TOK_SHL;
477 }
478 putChar(c);
479 return TOK_LT;
481 case '>':
482 c = getChar();
483 if (c == '=')
484 return TOK_GE;
485 if (c == '>') {
486 c = getChar();
487 if (c == '>') {
488 c = getChar();
489 if (c == '=')
490 return TOK_ASSIGN_USHR;
491 putChar(c);
492 return TOK_USHR;
493 }
494 return TOK_SHR;
495 }
496 putChar(c);
497 return TOK_GT;
499 case ' ':
500 case '\t':
501 case '\f':
502 break;
504 case '"':
505 return stringLiteral();
507 default:
508 if (c >= '0' && c <= '9')
509 return numberLiteral(c);
510 if (c == '_' || (isascii(c) && isalpha(c)))
511 return identifier(c);
512 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
513 return TOK_ERROR;
514 }
515 }
516 }
518 TokenKind
519 Scanner::scan()
520 {
521 if (pushed_) {
522 pushed_ = false;
523 return lastToken_;
524 }
525 lastToken_ = next();
526 return lastToken_;
527 }
529 TokenKind
530 Scanner::peek()
531 {
532 if (pushed_)
533 return lastToken_;
534 scan();
535 pushed_ = true;
536 return lastToken_;
537 }