| author | David Anderson <dvander@alliedmods.net> |
| Wed Dec 26 17:14:12 2012 -0800 (2012-12-26) | |
| changeset 241 | 18f3010a358c |
| parent 196 | 9904633b1dfc |
| child 251 | ca99a81745fe |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "../Utility.h"
25 #include "../Heap-inl.h"
26 #include "Scanner.h"
28 using namespace ke;
30 const char *ke::TokenNames[] =
31 {
32 #define _(name, str) str,
33 TOKENMAP(_)
34 #undef _
35 NULL
36 };
38 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
39 : cc_(cc),
40 stream_(stream),
41 length_(length),
42 pushed_(false),
43 allowTags_(true),
44 saved_('\0'),
45 pos_(stream_)
46 {
47 cur_.line = 1;
48 cur_.col = 1;
49 last_ = cur_;
50 }
52 char
53 Scanner::getChar()
54 {
55 if (saved_ != '\0') {
56 char temp = saved_;
57 saved_ = '\0';
58 return temp;
59 }
60 if (pos_ >= stream_ + length_)
61 return '\0';
62 last_.col = cur_.col++;
63 return *pos_++;
64 }
66 void
67 Scanner::putChar(char c)
68 {
69 assert(saved_ == '\0');
70 saved_ = c;
71 cur_.col = last_.col;
72 }
74 void
75 Scanner::calcNewLine(char c)
76 {
77 cur_.line++;
78 cur_.col = 1;
79 last_ = cur_;
81 char o;
82 if (c == '\r' && ((o = getChar()) != '\n'))
83 putChar(o);
84 }
86 void
87 Scanner::eatSingleLine()
88 {
89 char c;
90 while ((c = getChar()) != '\0') {
91 if (c == '\n' || c == '\r') {
92 calcNewLine(c);
93 break;
94 }
95 }
96 }
98 bool
99 Scanner::eatComment()
100 {
101 char c;
103 while ((c = getChar()) != '\0') {
104 while (c == '*') {
105 c = getChar();
106 if (c == '\0')
107 return false;
108 if (c == '/')
109 return true;
110 }
111 if (c == '\r' || c == '\n')
112 calcNewLine(c);
113 }
115 return false;
116 }
118 static inline bool
119 IsDigit(char c)
120 {
121 return c >= '0' && c <= '9';
122 }
124 // Based off the logic in sc2.c's ftoi()...
125 double
126 Scanner::ParseDouble(const char *string)
127 {
128 const char *ptr = string;
130 double number = 0.0;
131 while (IsDigit(*ptr)) {
132 number = (number * 10) + (*ptr - '0');
133 ptr++;
134 }
136 assert(*ptr == '.');
137 ptr++;
139 double fraction = 0.0;
140 double multiplier = 1.0;
141 while (IsDigit(*ptr)) {
142 fraction = (fraction * 10) + (*ptr - '0');
143 multiplier = multiplier / 10.0;
144 ptr++;
145 }
147 number += fraction * multiplier;
149 if (*ptr++ == 'e') {
150 int sign = 1;
151 if (*ptr == '-') {
152 sign = -1;
153 ptr++;
154 }
156 int exponent = 0;
157 while (IsDigit(*ptr)) {
158 exponent = (exponent * 10) + (*ptr - '0');
159 ptr++;
160 }
162 multiplier = pow(10.0, exponent * sign);
163 number *= multiplier;
164 }
166 return number;
167 }
169 static inline bool
170 IsHexDigit(char c)
171 {
172 return IsDigit(c) ||
173 (c >= 'a' && c <= 'f') ||
174 (c >= 'A' && c <= 'F');
175 }
177 // Based on the logic for litchar() in sc2.c.
178 int
179 Scanner::readEscapeCode()
180 {
181 char c = getChar();
182 if (c == '\\')
183 return c;
185 switch (c) {
186 case 'a':
187 return '\a';
188 case 'b':
189 return '\b';
190 case 'e':
191 return 27; // Apparently \e is non-standard.
192 case 'f':
193 return '\f';
194 case 'n':
195 return '\n';
196 case 'r':
197 return '\r';
198 case 't':
199 return '\t';
200 case '\v':
201 return '\v';
203 case 'x':
204 {
205 unsigned digits = 0;
206 char r = 0;
208 c = getChar();
209 while (IsHexDigit(c) && digits < 2) {
210 if (IsDigit(c))
211 c = (c << 4) + (c - '0');
212 else
213 c = (c << 4) + (tolower(c) - 'a' + 10);
214 digits++;
215 c = getChar();
216 }
218 // Swallow a trailing ';'
219 if (c != ';')
220 putChar(c);
222 return r;
223 }
225 case '\'':
226 case '\"':
227 case '%':
228 return c;
230 default:
231 {
232 if (IsDigit(c)) {
233 // \ddd
234 char r = 0;
235 while (IsDigit(c)) {
236 r = r * 10 + (c - '0');
237 c = getChar();
238 }
240 // Swallow a trailing ;
241 if (c != ';')
242 putChar(c);
244 return r;
245 }
246 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
247 return INT_MAX;
248 }
249 }
250 }
252 TokenKind
253 Scanner::stringLiteral()
254 {
255 literal_.clear();
257 for (;;) {
258 char c = getChar();
259 if (c == '\"')
260 break;
261 if (c == '\r' || c == '\n' || c == '\0') {
262 cc_.reportError(begin(), Message_UnterminatedString);
263 return TOK_ERROR;
264 }
265 if (c == '\\') {
266 int code = readEscapeCode();
267 if (code == INT_MAX)
268 return TOK_ERROR;
269 c = char(code);
270 }
271 if (!literal_.append(c))
272 return TOK_ERROR;
273 }
275 if (!literal_.append('\0'))
276 return TOK_ERROR;
278 return TOK_STRING_LITERAL;
279 }
281 TokenKind
282 Scanner::hexLiteral()
283 {
284 literal_.clear();
286 char c;
287 for (;;) {
288 c = getChar();
289 if (!IsHexDigit(c))
290 break;
291 if (!literal_.append(c))
292 return TOK_ERROR;
293 }
295 putChar(c);
297 if (!literal_.append('\0'))
298 return TOK_ERROR;
299 return TOK_HEX_LITERAL;
300 }
302 TokenKind
303 Scanner::numberLiteral(char first)
304 {
305 literal_.clear();
306 if (!literal_.append(first))
307 return TOK_ERROR;
309 char c;
310 for (;;) {
311 c = getChar();
312 if (!IsDigit(c))
313 break;
314 if (!literal_.append(c))
315 return TOK_ERROR;
316 }
318 // Detect a hexadecimal string.
319 if (literal_.length() == 1 &&
320 literal_[0] == '0' &&
321 (c == 'x' || c == 'X'))
322 {
323 return hexLiteral();
324 }
326 if (c != '.') {
327 putChar(c);
328 if (!literal_.append('\0'))
329 return TOK_ERROR;
330 return TOK_INTEGER_LITERAL;
331 }
332 if (!literal_.append(c))
333 return TOK_ERROR;
335 c = getChar();
336 if (!IsDigit(c)) {
337 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
338 return TOK_ERROR;
339 }
340 if (!literal_.append(c))
341 return TOK_ERROR;
343 for (;;) {
344 c = getChar();
345 if (!IsDigit(c))
346 break;
347 if (!literal_.append(c))
348 return TOK_ERROR;
349 }
351 if (c != 'e') {
352 putChar(c);
353 } else {
354 if (!literal_.append(c))
355 return TOK_ERROR;
356 c = getChar();
357 if (c == '-') {
358 if (!literal_.append(c))
359 return TOK_ERROR;
360 c = getChar();
361 }
362 if (!IsDigit(c)) {
363 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
364 return TOK_ERROR;
365 }
366 if (!literal_.append(c))
367 return TOK_ERROR;
368 for (;;) {
369 c = getChar();
370 if (!IsDigit(c)) {
371 putChar(c);
372 break;
373 }
374 if (!literal_.append(c))
375 return TOK_ERROR;
376 }
377 }
379 if (!literal_.append('\0'))
380 return TOK_ERROR;
382 return TOK_FLOAT_LITERAL;
383 }
385 static inline TokenKind
386 FindIdentifier(const char *str)
387 {
388 if (!strcmp(str, "new"))
389 return TOK_NEW;
390 if (!strcmp(str, "for"))
391 return TOK_FOR;
392 if (!strcmp(str, "return"))
393 return TOK_RETURN;
394 if (!strcmp(str, "if"))
395 return TOK_IF;
396 if (!strcmp(str, "else"))
397 return TOK_ELSE;
398 if (!strcmp(str, "enum"))
399 return TOK_ENUM;
400 if (!strcmp(str, "while"))
401 return TOK_WHILE;
402 if (!strcmp(str, "do"))
403 return TOK_DO;
404 if (!strcmp(str, "break"))
405 return TOK_BREAK;
406 if (!strcmp(str, "continue"))
407 return TOK_CONTINUE;
408 if (!strcmp(str, "forward"))
409 return TOK_FORWARD;
410 if (!strcmp(str, "public"))
411 return TOK_PUBLIC;
412 if (!strcmp(str, "const"))
413 return TOK_CONST;
414 if (!strcmp(str, "native"))
415 return TOK_NATIVE;
416 if (!strcmp(str, "stock"))
417 return TOK_STOCK;
418 if (!strcmp(str, "true"))
419 return TOK_TRUE;
420 if (!strcmp(str, "false"))
421 return TOK_FALSE;
422 if (!strcmp(str, "struct"))
423 return TOK_STRUCT;
424 if (!strcmp(str, "switch"))
425 return TOK_SWITCH;
426 if (!strcmp(str, "case"))
427 return TOK_CASE;
428 if (!strcmp(str, "default"))
429 return TOK_DEFAULT;
430 if (!strcmp(str, "typedef"))
431 return TOK_TYPEDEF;
432 if (!strcmp(str, "void"))
433 return TOK_VOID;
434 if (!strcmp(str, "int"))
435 return TOK_INT;
436 if (!strcmp(str, "sizeof"))
437 return TOK_SIZEOF;
438 if (!strcmp(str, "import"))
439 return TOK_IMPORT;
440 return TOK_NONE;
441 }
443 TokenKind
444 Scanner::identifier(char first)
445 {
446 literal_.clear();
447 if (!literal_.append(first))
448 return TOK_ERROR;
450 char c;
451 for (;;) {
452 c = getChar();
453 if (!isascii(c) || (!isalnum(c) && c != '_'))
454 break;
455 if (!literal_.append(c))
456 return TOK_ERROR;
457 }
459 if (!literal_.append('\0'))
460 return TOK_ERROR;
462 // Most keywords do not act as labels.
463 TokenKind kind = FindIdentifier(literal());
464 if (kind != TOK_NONE) {
465 putChar(c);
466 return kind;
467 }
469 if (c == ':' && allowTags_)
470 return TOK_LABEL;
472 putChar(c);
474 // _ is a special case which acts as a tag and a keyword.
475 if (!strcmp(literal(), "_"))
476 return TOK_UNDERSCORE;
478 return TOK_NAME;
479 }
481 TokenKind
482 Scanner::next()
483 {
484 assert(!pushed_);
486 for (;;) {
487 begin_ = cur_;
489 char c = getChar();
490 switch (c) {
491 case '/':
492 {
493 c = getChar();
494 if (c == '/') {
495 eatSingleLine();
496 break;
497 }
498 if (c == '*') {
499 if (!eatComment()) {
500 cc_.reportError(begin(), Message_UnterminatedComment);
501 return TOK_ERROR;
502 }
503 break;
504 }
505 putChar(c);
506 return TOK_SLASH;
507 }
509 case '\n':
510 case '\r':
511 calcNewLine(c);
512 break;
514 case ';': return TOK_SEMICOLON;
515 case '{': return TOK_LBRACE;
516 case '}': return TOK_RBRACE;
517 case '(': return TOK_LPAREN;
518 case ')': return TOK_RPAREN;
519 case '[': return TOK_LBRACKET;
520 case ']': return TOK_RBRACKET;
521 case '~': return TOK_TILDE;
522 case '?': return TOK_QMARK;
523 case ':': return TOK_COLON;
525 case '.':
526 c = getChar();
527 if (c == '.') {
528 c = getChar();
529 if (c == '.')
530 return TOK_ELLIPSES;
531 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
532 return TOK_ERROR;
533 }
534 putChar(c);
535 return TOK_DOT;
537 case '*':
538 c = getChar();
539 if (c == '=')
540 return TOK_ASSIGN_MUL;
541 putChar(c);
542 return TOK_STAR;
544 case '+':
545 c = getChar();
546 if (c == '=')
547 return TOK_ASSIGN_ADD;
548 if (c == '+')
549 return TOK_INCREMENT;
550 putChar(c);
551 return TOK_PLUS;
553 case '&':
554 c = getChar();
555 if (c == '=')
556 return TOK_ASSIGN_BITAND;
557 if (c == '&')
558 return TOK_AND;
559 putChar(c);
560 return TOK_BITAND;
562 case '|':
563 c = getChar();
564 if (c == '=')
565 return TOK_ASSIGN_BITOR;
566 if (c == '|')
567 return TOK_OR;
568 putChar(c);
569 return TOK_BITOR;
571 case '^':
572 c = getChar();
573 if (c == '=')
574 return TOK_ASSIGN_BITXOR;
575 putChar(c);
576 return TOK_BITXOR;
578 case '%':
579 c = getChar();
580 if (c == '=')
581 return TOK_ASSIGN_MOD;
582 putChar(c);
583 return TOK_PERCENT;
585 case '-':
586 c = getChar();
587 if (c == '=')
588 return TOK_ASSIGN_SUB;
589 if (c == '-')
590 return TOK_DECREMENT;
591 putChar(c);
592 return TOK_MINUS;
594 case ',': return TOK_COMMA;
595 case '\0': return TOK_EOF;
597 case '!':
598 c = getChar();
599 if (c == '=')
600 return TOK_NOTEQUALS;
601 putChar(c);
602 return TOK_NOT;
604 case '=':
605 c = getChar();
606 if (c == '=')
607 return TOK_EQUALS;
608 putChar(c);
609 return TOK_ASSIGN;
612 case '<':
613 c = getChar();
614 if (c == '=')
615 return TOK_LE;
616 if (c == '<') {
617 c = getChar();
618 if (c == '=')
619 return TOK_ASSIGN_SHL;
620 putChar(c);
621 return TOK_SHL;
622 }
623 putChar(c);
624 return TOK_LT;
626 case '>':
627 c = getChar();
628 if (c == '=')
629 return TOK_GE;
630 if (c == '>') {
631 c = getChar();
632 if (c == '>') {
633 c = getChar();
634 if (c == '=')
635 return TOK_ASSIGN_USHR;
636 putChar(c);
637 return TOK_USHR;
638 }
639 return TOK_SHR;
640 }
641 putChar(c);
642 return TOK_GT;
644 case ' ':
645 case '\t':
646 case '\f':
647 break;
649 case '"':
650 return stringLiteral();
652 default:
653 if (c >= '0' && c <= '9')
654 return numberLiteral(c);
655 if (c == '_' || (isascii(c) && isalpha(c)))
656 return identifier(c);
657 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
658 return TOK_ERROR;
659 }
660 }
661 }
663 TokenKind
664 Scanner::scan()
665 {
666 if (pushed_) {
667 pushed_ = false;
668 return lastToken_;
669 }
670 lastToken_ = next();
671 return lastToken_;
672 }
674 TokenKind
675 Scanner::peek()
676 {
677 if (pushed_)
678 return lastToken_;
679 scan();
680 pushed_ = true;
681 return lastToken_;
682 }