| author | David Anderson <dvander@alliedmods.net> |
| Sat Sep 01 16:36:20 2012 -0700 (2012-09-01) | |
| changeset 174 | e718b56aba78 |
| parent 172 | 48cffae8c9e0 |
| child 176 | 2b9141b01023 |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "Scanner.h"
25 #include "Utility.h"
27 using namespace ke;
29 const char *ke::TokenNames[] =
30 {
31 #define _(name, str) str,
32 TOKENMAP(_)
33 #undef _
34 NULL
35 };
37 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
38 : cc_(cc),
39 stream_(stream),
40 length_(length),
41 pushed_(false),
42 allowTags_(true),
43 saved_('\0'),
44 pos_(stream_)
45 {
46 cur_.line = 1;
47 cur_.col = 1;
48 last_ = cur_;
49 }
51 char
52 Scanner::getChar()
53 {
54 if (saved_ != '\0') {
55 char temp = saved_;
56 saved_ = '\0';
57 return temp;
58 }
59 if (pos_ >= stream_ + length_)
60 return '\0';
61 last_.col = cur_.col++;
62 return *pos_++;
63 }
65 void
66 Scanner::putChar(char c)
67 {
68 assert(saved_ == '\0');
69 saved_ = c;
70 cur_.col = last_.col;
71 }
73 void
74 Scanner::calcNewLine(char c)
75 {
76 cur_.line++;
77 cur_.col = 1;
78 last_ = cur_;
80 char o;
81 if (c == '\r' && ((o = getChar()) != '\n'))
82 putChar(o);
83 }
85 void
86 Scanner::eatSingleLine()
87 {
88 char c;
89 while ((c = getChar()) != '\0') {
90 if (c == '\n' || c == '\r') {
91 calcNewLine(c);
92 break;
93 }
94 }
95 }
97 bool
98 Scanner::eatComment()
99 {
100 char c;
102 while ((c = getChar()) != '\0') {
103 while (c == '*') {
104 c = getChar();
105 if (c == '\0')
106 return false;
107 if (c == '/')
108 return true;
109 }
110 if (c == '\r' || c == '\n')
111 calcNewLine(c);
112 }
114 return false;
115 }
117 static inline bool
118 IsDigit(char c)
119 {
120 return c >= '0' && c <= '9';
121 }
123 // Based off the logic in sc2.c's ftoi()...
124 double
125 Scanner::ParseDouble(const char *string)
126 {
127 const char *ptr = string;
129 double number = 0.0;
130 while (IsDigit(*ptr)) {
131 number = (number * 10) + (*ptr - '0');
132 ptr++;
133 }
135 assert(*ptr == '.');
136 ptr++;
138 double fraction = 0.0;
139 double multiplier = 1.0;
140 while (IsDigit(*ptr)) {
141 fraction = (fraction * 10) + (*ptr - '0');
142 multiplier = multiplier / 10.0;
143 ptr++;
144 }
146 number += fraction * multiplier;
148 if (*ptr++ == 'e') {
149 int sign = 1;
150 if (*ptr == '-') {
151 sign = -1;
152 ptr++;
153 }
155 int exponent = 0;
156 while (IsDigit(*ptr)) {
157 exponent = (exponent * 10) + (*ptr - '0');
158 ptr++;
159 }
161 multiplier = pow(10.0, exponent * sign);
162 number *= multiplier;
163 }
165 return number;
166 }
168 static inline bool
169 IsHexDigit(char c)
170 {
171 return IsDigit(c) ||
172 (c >= 'a' && c <= 'f') ||
173 (c >= 'A' && c <= 'F');
174 }
176 // Based on the logic for litchar() in sc2.c.
177 int
178 Scanner::readEscapeCode()
179 {
180 char c = getChar();
181 if (c == '\\')
182 return c;
184 switch (c) {
185 case 'a':
186 return '\a';
187 case 'b':
188 return '\b';
189 case 'e':
190 return 27; // Apparently \e is non-standard.
191 case 'f':
192 return '\f';
193 case 'n':
194 return '\n';
195 case 'r':
196 return '\r';
197 case 't':
198 return '\t';
199 case '\v':
200 return '\v';
202 case 'x':
203 {
204 unsigned digits = 0;
205 char r = 0;
207 c = getChar();
208 while (IsHexDigit(c) && digits < 2) {
209 if (IsDigit(c))
210 c = (c << 4) + (c - '0');
211 else
212 c = (c << 4) + (tolower(c) - 'a' + 10);
213 digits++;
214 c = getChar();
215 }
217 // Swallow a trailing ';'
218 if (c != ';')
219 putChar(c);
221 return r;
222 }
224 case '\'':
225 case '\"':
226 case '%':
227 return c;
229 default:
230 {
231 if (IsDigit(c)) {
232 // \ddd
233 char r = 0;
234 while (IsDigit(c)) {
235 r = r * 10 + (c - '0');
236 c = getChar();
237 }
239 // Swallow a trailing ;
240 if (c != ';')
241 putChar(c);
243 return r;
244 }
245 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
246 return INT_MAX;
247 }
248 }
249 }
251 TokenKind
252 Scanner::stringLiteral()
253 {
254 literal_.clear();
256 for (;;) {
257 char c = getChar();
258 if (c == '\"')
259 break;
260 if (c == '\r' || c == '\n' || c == '\0') {
261 cc_.reportError(begin(), Message_UnterminatedString);
262 return TOK_ERROR;
263 }
264 if (c == '\\') {
265 int code = readEscapeCode();
266 if (code == INT_MAX)
267 return TOK_ERROR;
268 c = char(code);
269 }
270 if (!literal_.append(c))
271 return TOK_ERROR;
272 }
274 if (!literal_.append('\0'))
275 return TOK_ERROR;
277 return TOK_STRING_LITERAL;
278 }
280 TokenKind
281 Scanner::numberLiteral(char first)
282 {
283 literal_.clear();
284 if (!literal_.append(first))
285 return TOK_ERROR;
287 char c;
288 for (;;) {
289 c = getChar();
290 if (!IsDigit(c))
291 break;
292 if (!literal_.append(c))
293 return TOK_ERROR;
294 }
296 if (c != '.') {
297 putChar(c);
298 if (!literal_.append('\0'))
299 return TOK_ERROR;
300 return TOK_INTEGER_LITERAL;
301 }
302 if (!literal_.append(c))
303 return TOK_ERROR;
305 c = getChar();
306 if (!IsDigit(c)) {
307 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
308 return TOK_ERROR;
309 }
310 if (!literal_.append(c))
311 return TOK_ERROR;
313 for (;;) {
314 c = getChar();
315 if (!IsDigit(c))
316 break;
317 if (!literal_.append(c))
318 return TOK_ERROR;
319 }
321 if (c != 'e') {
322 putChar(c);
323 } else {
324 if (!literal_.append(c))
325 return TOK_ERROR;
326 c = getChar();
327 if (c == '-') {
328 if (!literal_.append(c))
329 return TOK_ERROR;
330 c = getChar();
331 }
332 if (!IsDigit(c)) {
333 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
334 return TOK_ERROR;
335 }
336 if (!literal_.append(c))
337 return TOK_ERROR;
338 for (;;) {
339 c = getChar();
340 if (!IsDigit(c)) {
341 putChar(c);
342 break;
343 }
344 if (!literal_.append(c))
345 return TOK_ERROR;
346 }
347 }
349 if (!literal_.append('\0'))
350 return TOK_ERROR;
352 return TOK_FLOAT_LITERAL;
353 }
355 static inline TokenKind
356 FindIdentifier(const char *str)
357 {
358 if (!strcmp(str, "new"))
359 return TOK_NEW;
360 if (!strcmp(str, "for"))
361 return TOK_FOR;
362 if (!strcmp(str, "return"))
363 return TOK_RETURN;
364 if (!strcmp(str, "if"))
365 return TOK_IF;
366 if (!strcmp(str, "else"))
367 return TOK_ELSE;
368 if (!strcmp(str, "enum"))
369 return TOK_ENUM;
370 if (!strcmp(str, "while"))
371 return TOK_WHILE;
372 if (!strcmp(str, "do"))
373 return TOK_DO;
374 if (!strcmp(str, "break"))
375 return TOK_BREAK;
376 if (!strcmp(str, "continue"))
377 return TOK_CONTINUE;
378 if (!strcmp(str, "forward"))
379 return TOK_FORWARD;
380 if (!strcmp(str, "public"))
381 return TOK_PUBLIC;
382 if (!strcmp(str, "const"))
383 return TOK_CONST;
384 if (!strcmp(str, "native"))
385 return TOK_NATIVE;
386 if (!strcmp(str, "stock"))
387 return TOK_STOCK;
388 if (!strcmp(str, "true"))
389 return TOK_TRUE;
390 if (!strcmp(str, "false"))
391 return TOK_FALSE;
392 if (!strcmp(str, "struct"))
393 return TOK_STRUCT;
394 if (!strcmp(str, "switch"))
395 return TOK_SWITCH;
396 if (!strcmp(str, "case"))
397 return TOK_CASE;
398 if (!strcmp(str, "default"))
399 return TOK_DEFAULT;
400 if (!strcmp(str, "typedef"))
401 return TOK_TYPEDEF;
402 if (!strcmp(str, "void"))
403 return TOK_VOID;
404 if (!strcmp(str, "int"))
405 return TOK_INT;
406 return TOK_NONE;
407 }
409 TokenKind
410 Scanner::identifier(char first)
411 {
412 literal_.clear();
413 if (!literal_.append(first))
414 return TOK_ERROR;
416 char c;
417 for (;;) {
418 c = getChar();
419 if (!isascii(c) || (!isalnum(c) && c != '_'))
420 break;
421 if (!literal_.append(c))
422 return TOK_ERROR;
423 }
425 if (!literal_.append('\0'))
426 return TOK_ERROR;
428 // Most keywords do not act as labels.
429 TokenKind kind = FindIdentifier(literal());
430 if (kind != TOK_NONE) {
431 putChar(c);
432 return kind;
433 }
435 if (c == ':' && allowTags_)
436 return TOK_LABEL;
438 putChar(c);
440 // _ is a special case which acts as a tag and a keyword.
441 if (!strcmp(literal(), "_"))
442 return TOK_UNDERSCORE;
444 return TOK_NAME;
445 }
447 TokenKind
448 Scanner::next()
449 {
450 assert(!pushed_);
452 for (;;) {
453 begin_ = cur_;
455 char c = getChar();
456 switch (c) {
457 case '/':
458 {
459 c = getChar();
460 if (c == '/') {
461 eatSingleLine();
462 break;
463 }
464 if (c == '*') {
465 if (!eatComment()) {
466 cc_.reportError(begin(), Message_UnterminatedComment);
467 return TOK_ERROR;
468 }
469 break;
470 }
471 putChar(c);
472 return TOK_SLASH;
473 }
475 case '\n':
476 case '\r':
477 calcNewLine(c);
478 break;
480 case ';': return TOK_SEMICOLON;
481 case '{': return TOK_LBRACE;
482 case '}': return TOK_RBRACE;
483 case '(': return TOK_LPAREN;
484 case ')': return TOK_RPAREN;
485 case '[': return TOK_LBRACKET;
486 case ']': return TOK_RBRACKET;
487 case '~': return TOK_TILDE;
488 case '?': return TOK_QMARK;
489 case ':': return TOK_COLON;
491 case '.':
492 c = getChar();
493 if (c == '.') {
494 c = getChar();
495 if (c == '.')
496 return TOK_ELLIPSES;
497 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
498 return TOK_ERROR;
499 }
500 putChar(c);
501 return TOK_DOT;
503 case '*':
504 c = getChar();
505 if (c == '=')
506 return TOK_ASSIGN_MUL;
507 putChar(c);
508 return TOK_STAR;
510 case '+':
511 c = getChar();
512 if (c == '=')
513 return TOK_ASSIGN_ADD;
514 if (c == '+')
515 return TOK_INCREMENT;
516 putChar(c);
517 return TOK_PLUS;
519 case '&':
520 c = getChar();
521 if (c == '=')
522 return TOK_ASSIGN_BITAND;
523 if (c == '&')
524 return TOK_AND;
525 putChar(c);
526 return TOK_BITAND;
528 case '|':
529 c = getChar();
530 if (c == '=')
531 return TOK_ASSIGN_BITOR;
532 if (c == '|')
533 return TOK_OR;
534 putChar(c);
535 return TOK_BITOR;
537 case '^':
538 c = getChar();
539 if (c == '=')
540 return TOK_ASSIGN_BITXOR;
541 putChar(c);
542 return TOK_BITXOR;
544 case '%':
545 c = getChar();
546 if (c == '=')
547 return TOK_ASSIGN_MOD;
548 putChar(c);
549 return TOK_PERCENT;
551 case '-':
552 c = getChar();
553 if (c == '=')
554 return TOK_ASSIGN_SUB;
555 if (c == '-')
556 return TOK_DECREMENT;
557 putChar(c);
558 return TOK_MINUS;
560 case ',': return TOK_COMMA;
561 case '\0': return TOK_EOF;
563 case '!':
564 c = getChar();
565 if (c == '=')
566 return TOK_NOTEQUALS;
567 putChar(c);
568 return TOK_NOT;
570 case '=':
571 c = getChar();
572 if (c == '=')
573 return TOK_EQUALS;
574 putChar(c);
575 return TOK_ASSIGN;
578 case '<':
579 c = getChar();
580 if (c == '=')
581 return TOK_LE;
582 if (c == '<') {
583 c = getChar();
584 if (c == '=')
585 return TOK_ASSIGN_SHL;
586 putChar(c);
587 return TOK_SHL;
588 }
589 putChar(c);
590 return TOK_LT;
592 case '>':
593 c = getChar();
594 if (c == '=')
595 return TOK_GE;
596 if (c == '>') {
597 c = getChar();
598 if (c == '>') {
599 c = getChar();
600 if (c == '=')
601 return TOK_ASSIGN_USHR;
602 putChar(c);
603 return TOK_USHR;
604 }
605 return TOK_SHR;
606 }
607 putChar(c);
608 return TOK_GT;
610 case ' ':
611 case '\t':
612 case '\f':
613 break;
615 case '"':
616 return stringLiteral();
618 default:
619 if (c >= '0' && c <= '9')
620 return numberLiteral(c);
621 if (c == '_' || (isascii(c) && isalpha(c)))
622 return identifier(c);
623 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
624 return TOK_ERROR;
625 }
626 }
627 }
629 TokenKind
630 Scanner::scan()
631 {
632 if (pushed_) {
633 pushed_ = false;
634 return lastToken_;
635 }
636 lastToken_ = next();
637 return lastToken_;
638 }
640 TokenKind
641 Scanner::peek()
642 {
643 if (pushed_)
644 return lastToken_;
645 scan();
646 pushed_ = true;
647 return lastToken_;
648 }