| author | Asher Baker <asherkin@gmail.com> |
| Mon Jun 10 15:52:44 2013 +0100 (2013-06-10) | |
| changeset 265 | 3a903b89a514 |
| parent 263 | ba85a47ee414 |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "../Utility.h"
25 #include "../Heap-inl.h"
26 #include "../Handles-inl.h"
27 #include "Scanner.h"
29 using namespace ke;
31 const char *ke::TokenNames[] =
32 {
33 #define _(name, str) str,
34 TOKENMAP(_)
35 #undef _
36 NULL
37 };
39 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
40 : cc_(cc),
41 stream_(stream),
42 length_(length),
43 pushed_(false),
44 allowTags_(true),
45 saved_('\0'),
46 pos_(stream_)
47 {
48 cur_.line = 1;
49 cur_.col = 1;
50 last_ = cur_;
51 }
53 char
54 Scanner::getChar()
55 {
56 if (saved_ != '\0') {
57 char temp = saved_;
58 saved_ = '\0';
59 return temp;
60 }
61 if (pos_ >= stream_ + length_)
62 return '\0';
63 last_.col = cur_.col++;
64 return *pos_++;
65 }
67 void
68 Scanner::putChar(char c)
69 {
70 assert(saved_ == '\0');
71 saved_ = c;
72 cur_.col = last_.col;
73 }
75 void
76 Scanner::calcNewLine(char c)
77 {
78 cur_.line++;
79 cur_.col = 1;
80 last_ = cur_;
82 char o;
83 if (c == '\r' && ((o = getChar()) != '\n'))
84 putChar(o);
85 }
87 void
88 Scanner::eatSingleLine()
89 {
90 char c;
91 while ((c = getChar()) != '\0') {
92 if (c == '\n' || c == '\r') {
93 calcNewLine(c);
94 break;
95 }
96 }
97 }
99 bool
100 Scanner::eatComment()
101 {
102 char c;
104 while ((c = getChar()) != '\0') {
105 while (c == '*') {
106 c = getChar();
107 if (c == '\0')
108 return false;
109 if (c == '/')
110 return true;
111 }
112 if (c == '\r' || c == '\n')
113 calcNewLine(c);
114 }
116 return false;
117 }
119 static inline bool
120 IsDigit(char c)
121 {
122 return c >= '0' && c <= '9';
123 }
125 // Based off the logic in sc2.c's ftoi()...
126 double
127 Scanner::ParseDouble(const char *string)
128 {
129 const char *ptr = string;
131 double number = 0.0;
132 while (IsDigit(*ptr)) {
133 number = (number * 10) + (*ptr - '0');
134 ptr++;
135 }
137 assert(*ptr == '.');
138 ptr++;
140 double fraction = 0.0;
141 double multiplier = 1.0;
142 while (IsDigit(*ptr)) {
143 fraction = (fraction * 10) + (*ptr - '0');
144 multiplier = multiplier / 10.0;
145 ptr++;
146 }
148 number += fraction * multiplier;
150 if (*ptr++ == 'e') {
151 int sign = 1;
152 if (*ptr == '-') {
153 sign = -1;
154 ptr++;
155 }
157 int exponent = 0;
158 while (IsDigit(*ptr)) {
159 exponent = (exponent * 10) + (*ptr - '0');
160 ptr++;
161 }
163 multiplier = pow(10.0, exponent * sign);
164 number *= multiplier;
165 }
167 return number;
168 }
170 static inline bool
171 IsHexDigit(char c)
172 {
173 return IsDigit(c) ||
174 (c >= 'a' && c <= 'f') ||
175 (c >= 'A' && c <= 'F');
176 }
178 // Based on the logic for litchar() in sc2.c.
179 int
180 Scanner::readEscapeCode()
181 {
182 char c = getChar();
183 if (c == '\\')
184 return c;
186 switch (c) {
187 case 'a':
188 return '\a';
189 case 'b':
190 return '\b';
191 case 'e':
192 return 27; // Apparently \e is non-standard.
193 case 'f':
194 return '\f';
195 case 'n':
196 return '\n';
197 case 'r':
198 return '\r';
199 case 't':
200 return '\t';
201 case 'v':
202 return '\v';
204 case 'x':
205 {
206 unsigned digits = 0;
207 char r = 0;
209 c = getChar();
210 while (IsHexDigit(c) && digits < 2) {
211 if (IsDigit(c))
212 c = (c << 4) + (c - '0');
213 else
214 c = (c << 4) + (tolower(c) - 'a' + 10);
215 digits++;
216 c = getChar();
217 }
219 // Swallow a trailing ';'
220 if (c != ';')
221 putChar(c);
223 return r;
224 }
226 case '\'':
227 case '\"':
228 case '%':
229 return c;
231 default:
232 {
233 if (IsDigit(c)) {
234 // \ddd
235 char r = 0;
236 while (IsDigit(c)) {
237 r = r * 10 + (c - '0');
238 c = getChar();
239 }
241 // Swallow a trailing ;
242 if (c != ';')
243 putChar(c);
245 return r;
246 }
247 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
248 return INT_MAX;
249 }
250 }
251 }
253 TokenKind
254 Scanner::stringLiteral()
255 {
256 literal_.clear();
258 for (;;) {
259 char c = getChar();
260 if (c == '\"')
261 break;
262 if (c == '\r' || c == '\n' || c == '\0') {
263 cc_.reportError(begin(), Message_UnterminatedString);
264 return TOK_ERROR;
265 }
266 if (c == '\\') {
267 int code = readEscapeCode();
268 if (code == INT_MAX)
269 return TOK_ERROR;
270 c = char(code);
271 }
272 if (!literal_.append(c))
273 return TOK_ERROR;
274 }
276 if (!literal_.append('\0'))
277 return TOK_ERROR;
279 return TOK_STRING_LITERAL;
280 }
282 TokenKind
283 Scanner::hexLiteral()
284 {
285 literal_.clear();
287 char c;
288 for (;;) {
289 c = getChar();
290 if (!IsHexDigit(c))
291 break;
292 if (!literal_.append(c))
293 return TOK_ERROR;
294 }
296 putChar(c);
298 if (!literal_.append('\0'))
299 return TOK_ERROR;
300 return TOK_HEX_LITERAL;
301 }
303 TokenKind
304 Scanner::numberLiteral(char first)
305 {
306 literal_.clear();
307 if (!literal_.append(first))
308 return TOK_ERROR;
310 char c;
311 for (;;) {
312 c = getChar();
313 if (!IsDigit(c))
314 break;
315 if (!literal_.append(c))
316 return TOK_ERROR;
317 }
319 // Detect a hexadecimal string.
320 if (literal_.length() == 1 &&
321 literal_[0] == '0' &&
322 (c == 'x' || c == 'X'))
323 {
324 return hexLiteral();
325 }
327 if (c != '.') {
328 putChar(c);
329 if (!literal_.append('\0'))
330 return TOK_ERROR;
331 return TOK_INTEGER_LITERAL;
332 }
333 if (!literal_.append(c))
334 return TOK_ERROR;
336 c = getChar();
337 if (!IsDigit(c)) {
338 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
339 return TOK_ERROR;
340 }
341 if (!literal_.append(c))
342 return TOK_ERROR;
344 for (;;) {
345 c = getChar();
346 if (!IsDigit(c))
347 break;
348 if (!literal_.append(c))
349 return TOK_ERROR;
350 }
352 if (c != 'e') {
353 putChar(c);
354 } else {
355 if (!literal_.append(c))
356 return TOK_ERROR;
357 c = getChar();
358 if (c == '-') {
359 if (!literal_.append(c))
360 return TOK_ERROR;
361 c = getChar();
362 }
363 if (!IsDigit(c)) {
364 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
365 return TOK_ERROR;
366 }
367 if (!literal_.append(c))
368 return TOK_ERROR;
369 for (;;) {
370 c = getChar();
371 if (!IsDigit(c)) {
372 putChar(c);
373 break;
374 }
375 if (!literal_.append(c))
376 return TOK_ERROR;
377 }
378 }
380 if (!literal_.append('\0'))
381 return TOK_ERROR;
383 return TOK_FLOAT_LITERAL;
384 }
386 static inline TokenKind
387 FindIdentifier(const char *str)
388 {
389 if (!strcmp(str, "new"))
390 return TOK_NEW;
391 if (!strcmp(str, "for"))
392 return TOK_FOR;
393 if (!strcmp(str, "return"))
394 return TOK_RETURN;
395 if (!strcmp(str, "if"))
396 return TOK_IF;
397 if (!strcmp(str, "else"))
398 return TOK_ELSE;
399 if (!strcmp(str, "enum"))
400 return TOK_ENUM;
401 if (!strcmp(str, "while"))
402 return TOK_WHILE;
403 if (!strcmp(str, "do"))
404 return TOK_DO;
405 if (!strcmp(str, "break"))
406 return TOK_BREAK;
407 if (!strcmp(str, "continue"))
408 return TOK_CONTINUE;
409 if (!strcmp(str, "forward"))
410 return TOK_FORWARD;
411 if (!strcmp(str, "public"))
412 return TOK_PUBLIC;
413 if (!strcmp(str, "const"))
414 return TOK_CONST;
415 if (!strcmp(str, "native"))
416 return TOK_NATIVE;
417 if (!strcmp(str, "stock"))
418 return TOK_STOCK;
419 if (!strcmp(str, "true"))
420 return TOK_TRUE;
421 if (!strcmp(str, "false"))
422 return TOK_FALSE;
423 if (!strcmp(str, "struct"))
424 return TOK_STRUCT;
425 if (!strcmp(str, "switch"))
426 return TOK_SWITCH;
427 if (!strcmp(str, "case"))
428 return TOK_CASE;
429 if (!strcmp(str, "default"))
430 return TOK_DEFAULT;
431 if (!strcmp(str, "typedef"))
432 return TOK_TYPEDEF;
433 if (!strcmp(str, "void"))
434 return TOK_VOID;
435 if (!strcmp(str, "int"))
436 return TOK_INT;
437 if (!strcmp(str, "sizeof"))
438 return TOK_SIZEOF;
439 if (!strcmp(str, "import"))
440 return TOK_IMPORT;
441 if (!strcmp(str, "from"))
442 return TOK_FROM;
443 return TOK_NONE;
444 }
446 TokenKind
447 Scanner::identifier(char first)
448 {
449 literal_.clear();
450 if (!literal_.append(first))
451 return TOK_ERROR;
453 char c;
454 for (;;) {
455 c = getChar();
456 if (!isascii(c) || (!isalnum(c) && c != '_'))
457 break;
458 if (!literal_.append(c))
459 return TOK_ERROR;
460 }
462 if (!literal_.append('\0'))
463 return TOK_ERROR;
465 // Most keywords do not act as labels.
466 TokenKind kind = FindIdentifier(literal());
467 if (kind != TOK_NONE) {
468 putChar(c);
469 return kind;
470 }
472 if (c == ':' && allowTags_)
473 return TOK_LABEL;
475 putChar(c);
477 // _ is a special case which acts as a tag and a keyword.
478 if (!strcmp(literal(), "_"))
479 return TOK_UNDERSCORE;
481 return TOK_NAME;
482 }
484 TokenKind
485 Scanner::next()
486 {
487 assert(!pushed_);
489 for (;;) {
490 begin_ = cur_;
492 char c = getChar();
493 switch (c) {
494 case '/':
495 {
496 c = getChar();
497 if (c == '/') {
498 eatSingleLine();
499 break;
500 }
501 if (c == '*') {
502 if (!eatComment()) {
503 cc_.reportError(begin(), Message_UnterminatedComment);
504 return TOK_ERROR;
505 }
506 break;
507 }
508 putChar(c);
509 return TOK_SLASH;
510 }
512 case '\n':
513 case '\r':
514 calcNewLine(c);
515 break;
517 case ';': return TOK_SEMICOLON;
518 case '{': return TOK_LBRACE;
519 case '}': return TOK_RBRACE;
520 case '(': return TOK_LPAREN;
521 case ')': return TOK_RPAREN;
522 case '[': return TOK_LBRACKET;
523 case ']': return TOK_RBRACKET;
524 case '~': return TOK_TILDE;
525 case '?': return TOK_QMARK;
526 case ':': return TOK_COLON;
528 case '.':
529 c = getChar();
530 if (c == '.') {
531 c = getChar();
532 if (c == '.')
533 return TOK_ELLIPSES;
534 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
535 return TOK_ERROR;
536 }
537 putChar(c);
538 return TOK_DOT;
540 case '*':
541 c = getChar();
542 if (c == '=')
543 return TOK_ASSIGN_MUL;
544 putChar(c);
545 return TOK_STAR;
547 case '+':
548 c = getChar();
549 if (c == '=')
550 return TOK_ASSIGN_ADD;
551 if (c == '+')
552 return TOK_INCREMENT;
553 putChar(c);
554 return TOK_PLUS;
556 case '&':
557 c = getChar();
558 if (c == '=')
559 return TOK_ASSIGN_BITAND;
560 if (c == '&')
561 return TOK_AND;
562 putChar(c);
563 return TOK_BITAND;
565 case '|':
566 c = getChar();
567 if (c == '=')
568 return TOK_ASSIGN_BITOR;
569 if (c == '|')
570 return TOK_OR;
571 putChar(c);
572 return TOK_BITOR;
574 case '^':
575 c = getChar();
576 if (c == '=')
577 return TOK_ASSIGN_BITXOR;
578 putChar(c);
579 return TOK_BITXOR;
581 case '%':
582 c = getChar();
583 if (c == '=')
584 return TOK_ASSIGN_MOD;
585 putChar(c);
586 return TOK_PERCENT;
588 case '-':
589 c = getChar();
590 if (c == '=')
591 return TOK_ASSIGN_SUB;
592 if (c == '-')
593 return TOK_DECREMENT;
594 putChar(c);
595 return TOK_MINUS;
597 case ',': return TOK_COMMA;
598 case '\0': return TOK_EOF;
600 case '!':
601 c = getChar();
602 if (c == '=')
603 return TOK_NOTEQUALS;
604 putChar(c);
605 return TOK_NOT;
607 case '=':
608 c = getChar();
609 if (c == '=')
610 return TOK_EQUALS;
611 putChar(c);
612 return TOK_ASSIGN;
615 case '<':
616 c = getChar();
617 if (c == '=')
618 return TOK_LE;
619 if (c == '<') {
620 c = getChar();
621 if (c == '=')
622 return TOK_ASSIGN_SHL;
623 putChar(c);
624 return TOK_SHL;
625 }
626 putChar(c);
627 return TOK_LT;
629 case '>':
630 c = getChar();
631 if (c == '=')
632 return TOK_GE;
633 if (c == '>') {
634 c = getChar();
635 if (c == '>') {
636 c = getChar();
637 if (c == '=')
638 return TOK_ASSIGN_USHR;
639 putChar(c);
640 return TOK_USHR;
641 }
642 return TOK_SHR;
643 }
644 putChar(c);
645 return TOK_GT;
647 case ' ':
648 case '\t':
649 case '\f':
650 break;
652 case '"':
653 return stringLiteral();
655 default:
656 if (c >= '0' && c <= '9')
657 return numberLiteral(c);
658 if (c == '_' || (isascii(c) && isalpha(c)))
659 return identifier(c);
660 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
661 return TOK_ERROR;
662 }
663 }
664 }
666 TokenKind
667 Scanner::scan()
668 {
669 if (pushed_) {
670 pushed_ = false;
671 return lastToken_;
672 }
673 lastToken_ = next();
674 return lastToken_;
675 }
677 TokenKind
678 Scanner::peek()
679 {
680 if (pushed_)
681 return lastToken_;
682 scan();
683 pushed_ = true;
684 return lastToken_;
685 }