| author | David Anderson <dvander@alliedmods.net> |
| Tue Jan 01 20:07:09 2013 -0800 (2013-01-01) | |
| changeset 254 | cd167be62cbe |
| parent 251 | ca99a81745fe |
| child 258 | 241d082d6d89 |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "../Utility.h"
25 #include "../Heap-inl.h"
26 #include "../Handles-inl.h"
27 #include "Scanner.h"
29 using namespace ke;
31 const char *ke::TokenNames[] =
32 {
33 #define _(name, str) str,
34 TOKENMAP(_)
35 #undef _
36 NULL
37 };
39 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
40 : cc_(cc),
41 stream_(stream),
42 length_(length),
43 pushed_(false),
44 allowTags_(true),
45 saved_('\0'),
46 pos_(stream_)
47 {
48 cur_.line = 1;
49 cur_.col = 1;
50 last_ = cur_;
51 }
53 char
54 Scanner::getChar()
55 {
56 if (saved_ != '\0') {
57 char temp = saved_;
58 saved_ = '\0';
59 return temp;
60 }
61 if (pos_ >= stream_ + length_)
62 return '\0';
63 last_.col = cur_.col++;
64 return *pos_++;
65 }
67 void
68 Scanner::putChar(char c)
69 {
70 assert(saved_ == '\0');
71 saved_ = c;
72 cur_.col = last_.col;
73 }
75 void
76 Scanner::calcNewLine(char c)
77 {
78 cur_.line++;
79 cur_.col = 1;
80 last_ = cur_;
82 char o;
83 if (c == '\r' && ((o = getChar()) != '\n'))
84 putChar(o);
85 }
87 void
88 Scanner::eatSingleLine()
89 {
90 char c;
91 while ((c = getChar()) != '\0') {
92 if (c == '\n' || c == '\r') {
93 calcNewLine(c);
94 break;
95 }
96 }
97 }
99 bool
100 Scanner::eatComment()
101 {
102 char c;
104 while ((c = getChar()) != '\0') {
105 while (c == '*') {
106 c = getChar();
107 if (c == '\0')
108 return false;
109 if (c == '/')
110 return true;
111 }
112 if (c == '\r' || c == '\n')
113 calcNewLine(c);
114 }
116 return false;
117 }
119 static inline bool
120 IsDigit(char c)
121 {
122 return c >= '0' && c <= '9';
123 }
125 // Based off the logic in sc2.c's ftoi()...
126 double
127 Scanner::ParseDouble(const char *string)
128 {
129 const char *ptr = string;
131 double number = 0.0;
132 while (IsDigit(*ptr)) {
133 number = (number * 10) + (*ptr - '0');
134 ptr++;
135 }
137 assert(*ptr == '.');
138 ptr++;
140 double fraction = 0.0;
141 double multiplier = 1.0;
142 while (IsDigit(*ptr)) {
143 fraction = (fraction * 10) + (*ptr - '0');
144 multiplier = multiplier / 10.0;
145 ptr++;
146 }
148 number += fraction * multiplier;
150 if (*ptr++ == 'e') {
151 int sign = 1;
152 if (*ptr == '-') {
153 sign = -1;
154 ptr++;
155 }
157 int exponent = 0;
158 while (IsDigit(*ptr)) {
159 exponent = (exponent * 10) + (*ptr - '0');
160 ptr++;
161 }
163 multiplier = pow(10.0, exponent * sign);
164 number *= multiplier;
165 }
167 return number;
168 }
170 static inline bool
171 IsHexDigit(char c)
172 {
173 return IsDigit(c) ||
174 (c >= 'a' && c <= 'f') ||
175 (c >= 'A' && c <= 'F');
176 }
178 // Based on the logic for litchar() in sc2.c.
179 int
180 Scanner::readEscapeCode()
181 {
182 char c = getChar();
183 if (c == '\\')
184 return c;
186 switch (c) {
187 case 'a':
188 return '\a';
189 case 'b':
190 return '\b';
191 case 'e':
192 return 27; // Apparently \e is non-standard.
193 case 'f':
194 return '\f';
195 case 'n':
196 return '\n';
197 case 'r':
198 return '\r';
199 case 't':
200 return '\t';
201 case 'v':
202 return '\v';
204 case 'x':
205 {
206 unsigned digits = 0;
207 char r = 0;
209 c = getChar();
210 while (IsHexDigit(c) && digits < 2) {
211 if (IsDigit(c))
212 c = (c << 4) + (c - '0');
213 else
214 c = (c << 4) + (tolower(c) - 'a' + 10);
215 digits++;
216 c = getChar();
217 }
219 // Swallow a trailing ';'
220 if (c != ';')
221 putChar(c);
223 return r;
224 }
226 case '\'':
227 case '\"':
228 case '%':
229 return c;
231 default:
232 {
233 if (IsDigit(c)) {
234 // \ddd
235 char r = 0;
236 while (IsDigit(c)) {
237 r = r * 10 + (c - '0');
238 c = getChar();
239 }
241 // Swallow a trailing ;
242 if (c != ';')
243 putChar(c);
245 return r;
246 }
247 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
248 return INT_MAX;
249 }
250 }
251 }
253 TokenKind
254 Scanner::stringLiteral()
255 {
256 literal_.clear();
258 for (;;) {
259 char c = getChar();
260 if (c == '\"')
261 break;
262 if (c == '\r' || c == '\n' || c == '\0') {
263 cc_.reportError(begin(), Message_UnterminatedString);
264 return TOK_ERROR;
265 }
266 if (c == '\\') {
267 int code = readEscapeCode();
268 if (code == INT_MAX)
269 return TOK_ERROR;
270 c = char(code);
271 }
272 if (!literal_.append(c))
273 return TOK_ERROR;
274 }
276 if (!literal_.append('\0'))
277 return TOK_ERROR;
279 return TOK_STRING_LITERAL;
280 }
282 TokenKind
283 Scanner::hexLiteral()
284 {
285 literal_.clear();
287 char c;
288 for (;;) {
289 c = getChar();
290 if (!IsHexDigit(c))
291 break;
292 if (!literal_.append(c))
293 return TOK_ERROR;
294 }
296 putChar(c);
298 if (!literal_.append('\0'))
299 return TOK_ERROR;
300 return TOK_HEX_LITERAL;
301 }
303 TokenKind
304 Scanner::numberLiteral(char first)
305 {
306 literal_.clear();
307 if (!literal_.append(first))
308 return TOK_ERROR;
310 char c;
311 for (;;) {
312 c = getChar();
313 if (!IsDigit(c))
314 break;
315 if (!literal_.append(c))
316 return TOK_ERROR;
317 }
319 // Detect a hexadecimal string.
320 if (literal_.length() == 1 &&
321 literal_[0] == '0' &&
322 (c == 'x' || c == 'X'))
323 {
324 return hexLiteral();
325 }
327 if (c != '.') {
328 putChar(c);
329 if (!literal_.append('\0'))
330 return TOK_ERROR;
331 return TOK_INTEGER_LITERAL;
332 }
333 if (!literal_.append(c))
334 return TOK_ERROR;
336 c = getChar();
337 if (!IsDigit(c)) {
338 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
339 return TOK_ERROR;
340 }
341 if (!literal_.append(c))
342 return TOK_ERROR;
344 for (;;) {
345 c = getChar();
346 if (!IsDigit(c))
347 break;
348 if (!literal_.append(c))
349 return TOK_ERROR;
350 }
352 if (c != 'e') {
353 putChar(c);
354 } else {
355 if (!literal_.append(c))
356 return TOK_ERROR;
357 c = getChar();
358 if (c == '-') {
359 if (!literal_.append(c))
360 return TOK_ERROR;
361 c = getChar();
362 }
363 if (!IsDigit(c)) {
364 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
365 return TOK_ERROR;
366 }
367 if (!literal_.append(c))
368 return TOK_ERROR;
369 for (;;) {
370 c = getChar();
371 if (!IsDigit(c)) {
372 putChar(c);
373 break;
374 }
375 if (!literal_.append(c))
376 return TOK_ERROR;
377 }
378 }
380 if (!literal_.append('\0'))
381 return TOK_ERROR;
383 return TOK_FLOAT_LITERAL;
384 }
386 static inline TokenKind
387 FindIdentifier(const char *str)
388 {
389 if (!strcmp(str, "new"))
390 return TOK_NEW;
391 if (!strcmp(str, "for"))
392 return TOK_FOR;
393 if (!strcmp(str, "return"))
394 return TOK_RETURN;
395 if (!strcmp(str, "if"))
396 return TOK_IF;
397 if (!strcmp(str, "else"))
398 return TOK_ELSE;
399 if (!strcmp(str, "enum"))
400 return TOK_ENUM;
401 if (!strcmp(str, "while"))
402 return TOK_WHILE;
403 if (!strcmp(str, "do"))
404 return TOK_DO;
405 if (!strcmp(str, "break"))
406 return TOK_BREAK;
407 if (!strcmp(str, "continue"))
408 return TOK_CONTINUE;
409 if (!strcmp(str, "forward"))
410 return TOK_FORWARD;
411 if (!strcmp(str, "public"))
412 return TOK_PUBLIC;
413 if (!strcmp(str, "const"))
414 return TOK_CONST;
415 if (!strcmp(str, "native"))
416 return TOK_NATIVE;
417 if (!strcmp(str, "stock"))
418 return TOK_STOCK;
419 if (!strcmp(str, "true"))
420 return TOK_TRUE;
421 if (!strcmp(str, "false"))
422 return TOK_FALSE;
423 if (!strcmp(str, "struct"))
424 return TOK_STRUCT;
425 if (!strcmp(str, "switch"))
426 return TOK_SWITCH;
427 if (!strcmp(str, "case"))
428 return TOK_CASE;
429 if (!strcmp(str, "default"))
430 return TOK_DEFAULT;
431 if (!strcmp(str, "typedef"))
432 return TOK_TYPEDEF;
433 if (!strcmp(str, "void"))
434 return TOK_VOID;
435 if (!strcmp(str, "int"))
436 return TOK_INT;
437 if (!strcmp(str, "sizeof"))
438 return TOK_SIZEOF;
439 if (!strcmp(str, "import"))
440 return TOK_IMPORT;
441 return TOK_NONE;
442 }
444 TokenKind
445 Scanner::identifier(char first)
446 {
447 literal_.clear();
448 if (!literal_.append(first))
449 return TOK_ERROR;
451 char c;
452 for (;;) {
453 c = getChar();
454 if (!isascii(c) || (!isalnum(c) && c != '_'))
455 break;
456 if (!literal_.append(c))
457 return TOK_ERROR;
458 }
460 if (!literal_.append('\0'))
461 return TOK_ERROR;
463 // Most keywords do not act as labels.
464 TokenKind kind = FindIdentifier(literal());
465 if (kind != TOK_NONE) {
466 putChar(c);
467 return kind;
468 }
470 if (c == ':' && allowTags_)
471 return TOK_LABEL;
473 putChar(c);
475 // _ is a special case which acts as a tag and a keyword.
476 if (!strcmp(literal(), "_"))
477 return TOK_UNDERSCORE;
479 return TOK_NAME;
480 }
482 TokenKind
483 Scanner::next()
484 {
485 assert(!pushed_);
487 for (;;) {
488 begin_ = cur_;
490 char c = getChar();
491 switch (c) {
492 case '/':
493 {
494 c = getChar();
495 if (c == '/') {
496 eatSingleLine();
497 break;
498 }
499 if (c == '*') {
500 if (!eatComment()) {
501 cc_.reportError(begin(), Message_UnterminatedComment);
502 return TOK_ERROR;
503 }
504 break;
505 }
506 putChar(c);
507 return TOK_SLASH;
508 }
510 case '\n':
511 case '\r':
512 calcNewLine(c);
513 break;
515 case ';': return TOK_SEMICOLON;
516 case '{': return TOK_LBRACE;
517 case '}': return TOK_RBRACE;
518 case '(': return TOK_LPAREN;
519 case ')': return TOK_RPAREN;
520 case '[': return TOK_LBRACKET;
521 case ']': return TOK_RBRACKET;
522 case '~': return TOK_TILDE;
523 case '?': return TOK_QMARK;
524 case ':': return TOK_COLON;
526 case '.':
527 c = getChar();
528 if (c == '.') {
529 c = getChar();
530 if (c == '.')
531 return TOK_ELLIPSES;
532 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
533 return TOK_ERROR;
534 }
535 putChar(c);
536 return TOK_DOT;
538 case '*':
539 c = getChar();
540 if (c == '=')
541 return TOK_ASSIGN_MUL;
542 putChar(c);
543 return TOK_STAR;
545 case '+':
546 c = getChar();
547 if (c == '=')
548 return TOK_ASSIGN_ADD;
549 if (c == '+')
550 return TOK_INCREMENT;
551 putChar(c);
552 return TOK_PLUS;
554 case '&':
555 c = getChar();
556 if (c == '=')
557 return TOK_ASSIGN_BITAND;
558 if (c == '&')
559 return TOK_AND;
560 putChar(c);
561 return TOK_BITAND;
563 case '|':
564 c = getChar();
565 if (c == '=')
566 return TOK_ASSIGN_BITOR;
567 if (c == '|')
568 return TOK_OR;
569 putChar(c);
570 return TOK_BITOR;
572 case '^':
573 c = getChar();
574 if (c == '=')
575 return TOK_ASSIGN_BITXOR;
576 putChar(c);
577 return TOK_BITXOR;
579 case '%':
580 c = getChar();
581 if (c == '=')
582 return TOK_ASSIGN_MOD;
583 putChar(c);
584 return TOK_PERCENT;
586 case '-':
587 c = getChar();
588 if (c == '=')
589 return TOK_ASSIGN_SUB;
590 if (c == '-')
591 return TOK_DECREMENT;
592 putChar(c);
593 return TOK_MINUS;
595 case ',': return TOK_COMMA;
596 case '\0': return TOK_EOF;
598 case '!':
599 c = getChar();
600 if (c == '=')
601 return TOK_NOTEQUALS;
602 putChar(c);
603 return TOK_NOT;
605 case '=':
606 c = getChar();
607 if (c == '=')
608 return TOK_EQUALS;
609 putChar(c);
610 return TOK_ASSIGN;
613 case '<':
614 c = getChar();
615 if (c == '=')
616 return TOK_LE;
617 if (c == '<') {
618 c = getChar();
619 if (c == '=')
620 return TOK_ASSIGN_SHL;
621 putChar(c);
622 return TOK_SHL;
623 }
624 putChar(c);
625 return TOK_LT;
627 case '>':
628 c = getChar();
629 if (c == '=')
630 return TOK_GE;
631 if (c == '>') {
632 c = getChar();
633 if (c == '>') {
634 c = getChar();
635 if (c == '=')
636 return TOK_ASSIGN_USHR;
637 putChar(c);
638 return TOK_USHR;
639 }
640 return TOK_SHR;
641 }
642 putChar(c);
643 return TOK_GT;
645 case ' ':
646 case '\t':
647 case '\f':
648 break;
650 case '"':
651 return stringLiteral();
653 default:
654 if (c >= '0' && c <= '9')
655 return numberLiteral(c);
656 if (c == '_' || (isascii(c) && isalpha(c)))
657 return identifier(c);
658 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
659 return TOK_ERROR;
660 }
661 }
662 }
664 TokenKind
665 Scanner::scan()
666 {
667 if (pushed_) {
668 pushed_ = false;
669 return lastToken_;
670 }
671 lastToken_ = next();
672 return lastToken_;
673 }
675 TokenKind
676 Scanner::peek()
677 {
678 if (pushed_)
679 return lastToken_;
680 scan();
681 pushed_ = true;
682 return lastToken_;
683 }