| author | David Anderson <dvander@alliedmods.net> |
| Sun Sep 16 21:59:26 2012 -0700 (2012-09-16) | |
| changeset 190 | dd04fce774cd |
| parent 176 | 2b9141b01023 |
| child 194 | b029d63565a2 |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "Scanner.h"
25 #include "Utility.h"
27 using namespace ke;
29 const char *ke::TokenNames[] =
30 {
31 #define _(name, str) str,
32 TOKENMAP(_)
33 #undef _
34 NULL
35 };
37 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
38 : cc_(cc),
39 stream_(stream),
40 length_(length),
41 pushed_(false),
42 allowTags_(true),
43 saved_('\0'),
44 pos_(stream_)
45 {
46 cur_.line = 1;
47 cur_.col = 1;
48 last_ = cur_;
49 }
51 char
52 Scanner::getChar()
53 {
54 if (saved_ != '\0') {
55 char temp = saved_;
56 saved_ = '\0';
57 return temp;
58 }
59 if (pos_ >= stream_ + length_)
60 return '\0';
61 last_.col = cur_.col++;
62 return *pos_++;
63 }
65 void
66 Scanner::putChar(char c)
67 {
68 assert(saved_ == '\0');
69 saved_ = c;
70 cur_.col = last_.col;
71 }
73 void
74 Scanner::calcNewLine(char c)
75 {
76 cur_.line++;
77 cur_.col = 1;
78 last_ = cur_;
80 char o;
81 if (c == '\r' && ((o = getChar()) != '\n'))
82 putChar(o);
83 }
85 void
86 Scanner::eatSingleLine()
87 {
88 char c;
89 while ((c = getChar()) != '\0') {
90 if (c == '\n' || c == '\r') {
91 calcNewLine(c);
92 break;
93 }
94 }
95 }
97 bool
98 Scanner::eatComment()
99 {
100 char c;
102 while ((c = getChar()) != '\0') {
103 while (c == '*') {
104 c = getChar();
105 if (c == '\0')
106 return false;
107 if (c == '/')
108 return true;
109 }
110 if (c == '\r' || c == '\n')
111 calcNewLine(c);
112 }
114 return false;
115 }
117 static inline bool
118 IsDigit(char c)
119 {
120 return c >= '0' && c <= '9';
121 }
123 // Based off the logic in sc2.c's ftoi()...
124 double
125 Scanner::ParseDouble(const char *string)
126 {
127 const char *ptr = string;
129 double number = 0.0;
130 while (IsDigit(*ptr)) {
131 number = (number * 10) + (*ptr - '0');
132 ptr++;
133 }
135 assert(*ptr == '.');
136 ptr++;
138 double fraction = 0.0;
139 double multiplier = 1.0;
140 while (IsDigit(*ptr)) {
141 fraction = (fraction * 10) + (*ptr - '0');
142 multiplier = multiplier / 10.0;
143 ptr++;
144 }
146 number += fraction * multiplier;
148 if (*ptr++ == 'e') {
149 int sign = 1;
150 if (*ptr == '-') {
151 sign = -1;
152 ptr++;
153 }
155 int exponent = 0;
156 while (IsDigit(*ptr)) {
157 exponent = (exponent * 10) + (*ptr - '0');
158 ptr++;
159 }
161 multiplier = pow(10.0, exponent * sign);
162 number *= multiplier;
163 }
165 return number;
166 }
168 static inline bool
169 IsHexDigit(char c)
170 {
171 return IsDigit(c) ||
172 (c >= 'a' && c <= 'f') ||
173 (c >= 'A' && c <= 'F');
174 }
176 // Based on the logic for litchar() in sc2.c.
177 int
178 Scanner::readEscapeCode()
179 {
180 char c = getChar();
181 if (c == '\\')
182 return c;
184 switch (c) {
185 case 'a':
186 return '\a';
187 case 'b':
188 return '\b';
189 case 'e':
190 return 27; // Apparently \e is non-standard.
191 case 'f':
192 return '\f';
193 case 'n':
194 return '\n';
195 case 'r':
196 return '\r';
197 case 't':
198 return '\t';
199 case '\v':
200 return '\v';
202 case 'x':
203 {
204 unsigned digits = 0;
205 char r = 0;
207 c = getChar();
208 while (IsHexDigit(c) && digits < 2) {
209 if (IsDigit(c))
210 c = (c << 4) + (c - '0');
211 else
212 c = (c << 4) + (tolower(c) - 'a' + 10);
213 digits++;
214 c = getChar();
215 }
217 // Swallow a trailing ';'
218 if (c != ';')
219 putChar(c);
221 return r;
222 }
224 case '\'':
225 case '\"':
226 case '%':
227 return c;
229 default:
230 {
231 if (IsDigit(c)) {
232 // \ddd
233 char r = 0;
234 while (IsDigit(c)) {
235 r = r * 10 + (c - '0');
236 c = getChar();
237 }
239 // Swallow a trailing ;
240 if (c != ';')
241 putChar(c);
243 return r;
244 }
245 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
246 return INT_MAX;
247 }
248 }
249 }
251 TokenKind
252 Scanner::stringLiteral()
253 {
254 literal_.clear();
256 for (;;) {
257 char c = getChar();
258 if (c == '\"')
259 break;
260 if (c == '\r' || c == '\n' || c == '\0') {
261 cc_.reportError(begin(), Message_UnterminatedString);
262 return TOK_ERROR;
263 }
264 if (c == '\\') {
265 int code = readEscapeCode();
266 if (code == INT_MAX)
267 return TOK_ERROR;
268 c = char(code);
269 }
270 if (!literal_.append(c))
271 return TOK_ERROR;
272 }
274 if (!literal_.append('\0'))
275 return TOK_ERROR;
277 return TOK_STRING_LITERAL;
278 }
280 TokenKind
281 Scanner::hexLiteral()
282 {
283 literal_.clear();
285 char c;
286 for (;;) {
287 c = getChar();
288 if (!IsHexDigit(c))
289 break;
290 if (!literal_.append(c))
291 return TOK_ERROR;
292 }
294 putChar(c);
296 if (!literal_.append('\0'))
297 return TOK_ERROR;
298 return TOK_HEX_LITERAL;
299 }
301 TokenKind
302 Scanner::numberLiteral(char first)
303 {
304 literal_.clear();
305 if (!literal_.append(first))
306 return TOK_ERROR;
308 char c;
309 for (;;) {
310 c = getChar();
311 if (!IsDigit(c))
312 break;
313 if (!literal_.append(c))
314 return TOK_ERROR;
315 }
317 // Detect a hexadecimal string.
318 if (literal_.length() == 1 &&
319 literal_[0] == '0' &&
320 (c == 'x' || c == 'X'))
321 {
322 return hexLiteral();
323 }
325 if (c != '.') {
326 putChar(c);
327 if (!literal_.append('\0'))
328 return TOK_ERROR;
329 return TOK_INTEGER_LITERAL;
330 }
331 if (!literal_.append(c))
332 return TOK_ERROR;
334 c = getChar();
335 if (!IsDigit(c)) {
336 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
337 return TOK_ERROR;
338 }
339 if (!literal_.append(c))
340 return TOK_ERROR;
342 for (;;) {
343 c = getChar();
344 if (!IsDigit(c))
345 break;
346 if (!literal_.append(c))
347 return TOK_ERROR;
348 }
350 if (c != 'e') {
351 putChar(c);
352 } else {
353 if (!literal_.append(c))
354 return TOK_ERROR;
355 c = getChar();
356 if (c == '-') {
357 if (!literal_.append(c))
358 return TOK_ERROR;
359 c = getChar();
360 }
361 if (!IsDigit(c)) {
362 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
363 return TOK_ERROR;
364 }
365 if (!literal_.append(c))
366 return TOK_ERROR;
367 for (;;) {
368 c = getChar();
369 if (!IsDigit(c)) {
370 putChar(c);
371 break;
372 }
373 if (!literal_.append(c))
374 return TOK_ERROR;
375 }
376 }
378 if (!literal_.append('\0'))
379 return TOK_ERROR;
381 return TOK_FLOAT_LITERAL;
382 }
384 static inline TokenKind
385 FindIdentifier(const char *str)
386 {
387 if (!strcmp(str, "new"))
388 return TOK_NEW;
389 if (!strcmp(str, "for"))
390 return TOK_FOR;
391 if (!strcmp(str, "return"))
392 return TOK_RETURN;
393 if (!strcmp(str, "if"))
394 return TOK_IF;
395 if (!strcmp(str, "else"))
396 return TOK_ELSE;
397 if (!strcmp(str, "enum"))
398 return TOK_ENUM;
399 if (!strcmp(str, "while"))
400 return TOK_WHILE;
401 if (!strcmp(str, "do"))
402 return TOK_DO;
403 if (!strcmp(str, "break"))
404 return TOK_BREAK;
405 if (!strcmp(str, "continue"))
406 return TOK_CONTINUE;
407 if (!strcmp(str, "forward"))
408 return TOK_FORWARD;
409 if (!strcmp(str, "public"))
410 return TOK_PUBLIC;
411 if (!strcmp(str, "const"))
412 return TOK_CONST;
413 if (!strcmp(str, "native"))
414 return TOK_NATIVE;
415 if (!strcmp(str, "stock"))
416 return TOK_STOCK;
417 if (!strcmp(str, "true"))
418 return TOK_TRUE;
419 if (!strcmp(str, "false"))
420 return TOK_FALSE;
421 if (!strcmp(str, "struct"))
422 return TOK_STRUCT;
423 if (!strcmp(str, "switch"))
424 return TOK_SWITCH;
425 if (!strcmp(str, "case"))
426 return TOK_CASE;
427 if (!strcmp(str, "default"))
428 return TOK_DEFAULT;
429 if (!strcmp(str, "typedef"))
430 return TOK_TYPEDEF;
431 if (!strcmp(str, "void"))
432 return TOK_VOID;
433 if (!strcmp(str, "int"))
434 return TOK_INT;
435 return TOK_NONE;
436 }
438 TokenKind
439 Scanner::identifier(char first)
440 {
441 literal_.clear();
442 if (!literal_.append(first))
443 return TOK_ERROR;
445 char c;
446 for (;;) {
447 c = getChar();
448 if (!isascii(c) || (!isalnum(c) && c != '_'))
449 break;
450 if (!literal_.append(c))
451 return TOK_ERROR;
452 }
454 if (!literal_.append('\0'))
455 return TOK_ERROR;
457 // Most keywords do not act as labels.
458 TokenKind kind = FindIdentifier(literal());
459 if (kind != TOK_NONE) {
460 putChar(c);
461 return kind;
462 }
464 if (c == ':' && allowTags_)
465 return TOK_LABEL;
467 putChar(c);
469 // _ is a special case which acts as a tag and a keyword.
470 if (!strcmp(literal(), "_"))
471 return TOK_UNDERSCORE;
473 return TOK_NAME;
474 }
476 TokenKind
477 Scanner::next()
478 {
479 assert(!pushed_);
481 for (;;) {
482 begin_ = cur_;
484 char c = getChar();
485 switch (c) {
486 case '/':
487 {
488 c = getChar();
489 if (c == '/') {
490 eatSingleLine();
491 break;
492 }
493 if (c == '*') {
494 if (!eatComment()) {
495 cc_.reportError(begin(), Message_UnterminatedComment);
496 return TOK_ERROR;
497 }
498 break;
499 }
500 putChar(c);
501 return TOK_SLASH;
502 }
504 case '\n':
505 case '\r':
506 calcNewLine(c);
507 break;
509 case ';': return TOK_SEMICOLON;
510 case '{': return TOK_LBRACE;
511 case '}': return TOK_RBRACE;
512 case '(': return TOK_LPAREN;
513 case ')': return TOK_RPAREN;
514 case '[': return TOK_LBRACKET;
515 case ']': return TOK_RBRACKET;
516 case '~': return TOK_TILDE;
517 case '?': return TOK_QMARK;
518 case ':': return TOK_COLON;
520 case '.':
521 c = getChar();
522 if (c == '.') {
523 c = getChar();
524 if (c == '.')
525 return TOK_ELLIPSES;
526 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
527 return TOK_ERROR;
528 }
529 putChar(c);
530 return TOK_DOT;
532 case '*':
533 c = getChar();
534 if (c == '=')
535 return TOK_ASSIGN_MUL;
536 putChar(c);
537 return TOK_STAR;
539 case '+':
540 c = getChar();
541 if (c == '=')
542 return TOK_ASSIGN_ADD;
543 if (c == '+')
544 return TOK_INCREMENT;
545 putChar(c);
546 return TOK_PLUS;
548 case '&':
549 c = getChar();
550 if (c == '=')
551 return TOK_ASSIGN_BITAND;
552 if (c == '&')
553 return TOK_AND;
554 putChar(c);
555 return TOK_BITAND;
557 case '|':
558 c = getChar();
559 if (c == '=')
560 return TOK_ASSIGN_BITOR;
561 if (c == '|')
562 return TOK_OR;
563 putChar(c);
564 return TOK_BITOR;
566 case '^':
567 c = getChar();
568 if (c == '=')
569 return TOK_ASSIGN_BITXOR;
570 putChar(c);
571 return TOK_BITXOR;
573 case '%':
574 c = getChar();
575 if (c == '=')
576 return TOK_ASSIGN_MOD;
577 putChar(c);
578 return TOK_PERCENT;
580 case '-':
581 c = getChar();
582 if (c == '=')
583 return TOK_ASSIGN_SUB;
584 if (c == '-')
585 return TOK_DECREMENT;
586 putChar(c);
587 return TOK_MINUS;
589 case ',': return TOK_COMMA;
590 case '\0': return TOK_EOF;
592 case '!':
593 c = getChar();
594 if (c == '=')
595 return TOK_NOTEQUALS;
596 putChar(c);
597 return TOK_NOT;
599 case '=':
600 c = getChar();
601 if (c == '=')
602 return TOK_EQUALS;
603 putChar(c);
604 return TOK_ASSIGN;
607 case '<':
608 c = getChar();
609 if (c == '=')
610 return TOK_LE;
611 if (c == '<') {
612 c = getChar();
613 if (c == '=')
614 return TOK_ASSIGN_SHL;
615 putChar(c);
616 return TOK_SHL;
617 }
618 putChar(c);
619 return TOK_LT;
621 case '>':
622 c = getChar();
623 if (c == '=')
624 return TOK_GE;
625 if (c == '>') {
626 c = getChar();
627 if (c == '>') {
628 c = getChar();
629 if (c == '=')
630 return TOK_ASSIGN_USHR;
631 putChar(c);
632 return TOK_USHR;
633 }
634 return TOK_SHR;
635 }
636 putChar(c);
637 return TOK_GT;
639 case ' ':
640 case '\t':
641 case '\f':
642 break;
644 case '"':
645 return stringLiteral();
647 default:
648 if (c >= '0' && c <= '9')
649 return numberLiteral(c);
650 if (c == '_' || (isascii(c) && isalpha(c)))
651 return identifier(c);
652 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
653 return TOK_ERROR;
654 }
655 }
656 }
658 TokenKind
659 Scanner::scan()
660 {
661 if (pushed_) {
662 pushed_ = false;
663 return lastToken_;
664 }
665 lastToken_ = next();
666 return lastToken_;
667 }
669 TokenKind
670 Scanner::peek()
671 {
672 if (pushed_)
673 return lastToken_;
674 scan();
675 pushed_ = true;
676 return lastToken_;
677 }