| author | David Anderson <dvander@alliedmods.net> |
| Mon Jul 30 22:24:57 2012 -0700 (2012-07-30) | |
| changeset 143 | 393f1b8ac8c3 |
| parent 142 | a000bafece09 |
| child 146 | dc7d3b7a918c |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "Scanner.h"
25 #include "Utility.h"
27 using namespace ke;
29 const char *ke::TokenNames[] =
30 {
31 #define _(name, str) str,
32 TOKENMAP(_)
33 #undef _
34 NULL
35 };
37 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
38 : cc_(cc),
39 stream_(stream),
40 length_(length),
41 pushed_(false),
42 saved_('\0'),
43 pos_(stream_)
44 {
45 cur_.line = 1;
46 cur_.col = 1;
47 last_ = cur_;
48 }
50 char
51 Scanner::getChar()
52 {
53 if (saved_ != '\0') {
54 char temp = saved_;
55 saved_ = '\0';
56 return temp;
57 }
58 if (pos_ >= stream_ + length_)
59 return '\0';
60 last_.col = cur_.col++;
61 return *pos_++;
62 }
64 void
65 Scanner::putChar(char c)
66 {
67 assert(saved_ == '\0');
68 saved_ = c;
69 cur_.col = last_.col;
70 }
72 void
73 Scanner::calcNewLine(char c)
74 {
75 cur_.line++;
76 cur_.col = 1;
77 last_ = cur_;
79 char o;
80 if (c == '\r' && ((o = getChar()) != '\n'))
81 putChar(o);
82 }
84 void
85 Scanner::eatSingleLine()
86 {
87 char c;
88 while ((c = getChar()) != '\0') {
89 if (c == '\n' || c == '\r') {
90 calcNewLine(c);
91 break;
92 }
93 }
94 }
96 bool
97 Scanner::eatComment()
98 {
99 char c;
101 while ((c = getChar()) != '\0') {
102 while (c == '*') {
103 c = getChar();
104 if (c == '\0')
105 return false;
106 if (c == '/')
107 return true;
108 }
109 if (c == '\r' || c == '\n')
110 calcNewLine(c);
111 }
113 return false;
114 }
116 static inline bool
117 IsDigit(char c)
118 {
119 return c >= '0' && c <= '9';
120 }
122 // Based off the logic in sc2.c's ftoi()...
123 double
124 Scanner::ParseDouble(const char *string)
125 {
126 const char *ptr = string;
128 double number = 0.0;
129 while (IsDigit(*ptr)) {
130 number = (number * 10) + (*ptr - '0');
131 ptr++;
132 }
134 assert(*ptr == '.');
135 ptr++;
137 double fraction = 0.0;
138 double multiplier = 1.0;
139 while (IsDigit(*ptr)) {
140 fraction = (fraction * 10) + (*ptr - '0');
141 multiplier = multiplier / 10.0;
142 ptr++;
143 }
145 number += fraction * multiplier;
147 if (*ptr++ == 'e') {
148 int sign = 1;
149 if (*ptr == '-') {
150 sign = -1;
151 ptr++;
152 }
154 int exponent = 0;
155 while (IsDigit(*ptr)) {
156 exponent = (exponent * 10) + (*ptr - '0');
157 ptr++;
158 }
160 multiplier = pow(10.0, exponent * sign);
161 number *= multiplier;
162 }
164 return number;
165 }
167 static inline bool
168 IsHexDigit(char c)
169 {
170 return IsDigit(c) ||
171 (c >= 'a' && c <= 'f') ||
172 (c >= 'A' && c <= 'F');
173 }
175 // Based on the logic for litchar() in sc2.c.
176 int
177 Scanner::readEscapeCode()
178 {
179 char c = getChar();
180 if (c == '\\')
181 return c;
183 switch (c) {
184 case 'a':
185 return '\a';
186 case 'b':
187 return '\b';
188 case 'e':
189 return 27; // Apparently \e is non-standard.
190 case 'f':
191 return '\f';
192 case 'n':
193 return '\n';
194 case 'r':
195 return '\r';
196 case 't':
197 return '\t';
198 case '\v':
199 return '\v';
201 case 'x':
202 {
203 unsigned digits = 0;
204 char r = 0;
206 c = getChar();
207 while (IsHexDigit(c) && digits < 2) {
208 if (IsDigit(c))
209 c = (c << 4) + (c - '0');
210 else
211 c = (c << 4) + (tolower(c) - 'a' + 10);
212 digits++;
213 c = getChar();
214 }
216 // Swallow a trailing ';'
217 if (c != ';')
218 putChar(c);
220 return r;
221 }
223 case '\'':
224 case '\"':
225 case '%':
226 return c;
228 default:
229 {
230 if (IsDigit(c)) {
231 // \ddd
232 char r = 0;
233 while (IsDigit(c)) {
234 r = r * 10 + (c - '0');
235 c = getChar();
236 }
238 // Swallow a trailing ;
239 if (c != ';')
240 putChar(c);
242 return r;
243 }
244 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
245 return INT_MAX;
246 }
247 }
248 }
250 TokenKind
251 Scanner::stringLiteral()
252 {
253 literal_.clear();
255 for (;;) {
256 char c = getChar();
257 if (c == '\"')
258 break;
259 if (c == '\r' || c == '\n' || c == '\0') {
260 cc_.reportError(begin(), Message_UnterminatedString);
261 return TOK_ERROR;
262 }
263 if (c == '\\') {
264 int code = readEscapeCode();
265 if (code == INT_MAX)
266 return TOK_ERROR;
267 c = char(code);
268 }
269 if (!literal_.append(c))
270 return TOK_ERROR;
271 }
273 if (!literal_.append('\0'))
274 return TOK_ERROR;
276 return TOK_STRING_LITERAL;
277 }
279 TokenKind
280 Scanner::numberLiteral(char first)
281 {
282 literal_.clear();
283 if (!literal_.append(first))
284 return TOK_ERROR;
286 char c;
287 for (;;) {
288 c = getChar();
289 if (!IsDigit(c))
290 break;
291 if (!literal_.append(c))
292 return TOK_ERROR;
293 }
295 if (c != '.') {
296 putChar(c);
297 if (!literal_.append('\0'))
298 return TOK_ERROR;
299 return TOK_INTEGER_LITERAL;
300 }
301 if (!literal_.append(c))
302 return TOK_ERROR;
304 c = getChar();
305 if (!IsDigit(c)) {
306 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
307 return TOK_ERROR;
308 }
309 if (!literal_.append(c))
310 return TOK_ERROR;
312 for (;;) {
313 c = getChar();
314 if (!IsDigit(c))
315 break;
316 if (!literal_.append(c))
317 return TOK_ERROR;
318 }
320 if (c != 'e') {
321 putChar(c);
322 } else {
323 if (!literal_.append(c))
324 return TOK_ERROR;
325 c = getChar();
326 if (c == '-') {
327 if (!literal_.append(c))
328 return TOK_ERROR;
329 c = getChar();
330 }
331 if (!IsDigit(c)) {
332 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
333 return TOK_ERROR;
334 }
335 if (!literal_.append(c))
336 return TOK_ERROR;
337 for (;;) {
338 c = getChar();
339 if (!IsDigit(c)) {
340 putChar(c);
341 break;
342 }
343 if (!literal_.append(c))
344 return TOK_ERROR;
345 }
346 }
348 if (!literal_.append('\0'))
349 return TOK_ERROR;
351 return TOK_FLOAT_LITERAL;
352 }
354 TokenKind
355 Scanner::identifier(char first)
356 {
357 literal_.clear();
358 if (!literal_.append(first))
359 return TOK_ERROR;
361 char c;
362 for (;;) {
363 c = getChar();
364 if (!isascii(c) || (!isalnum(c) && c != '_')) {
365 if (c != ':')
366 putChar(c);
367 break;
368 }
369 if (!literal_.append(c))
370 return TOK_ERROR;
371 }
373 if (!literal_.append('\0'))
374 return TOK_ERROR;
376 if (c != ':' && !strcmp(literal(), "_"))
377 return TOK_UNDERSCORE;
378 if (!strcmp(literal(), "new"))
379 return TOK_NEW;
380 if (!strcmp(literal(), "for"))
381 return TOK_FOR;
382 if (!strcmp(literal(), "return"))
383 return TOK_RETURN;
384 if (!strcmp(literal(), "if"))
385 return TOK_IF;
386 if (!strcmp(literal(), "else"))
387 return TOK_ELSE;
388 if (!strcmp(literal(), "enum"))
389 return TOK_ENUM;
390 if (!strcmp(literal(), "while"))
391 return TOK_WHILE;
392 if (!strcmp(literal(), "do"))
393 return TOK_DO;
394 if (!strcmp(literal(), "break"))
395 return TOK_BREAK;
396 if (!strcmp(literal(), "continue"))
397 return TOK_CONTINUE;
398 if (!strcmp(literal(), "forward"))
399 return TOK_FORWARD;
400 if (!strcmp(literal(), "public"))
401 return TOK_PUBLIC;
402 if (!strcmp(literal(), "const"))
403 return TOK_CONST;
404 if (!strcmp(literal(), "native"))
405 return TOK_NATIVE;
406 if (!strcmp(literal(), "stock"))
407 return TOK_STOCK;
408 if (!strcmp(literal(), "true"))
409 return TOK_TRUE;
410 if (!strcmp(literal(), "false"))
411 return TOK_FALSE;
412 return (c == ':') ? TOK_LABEL : TOK_NAME;
413 }
415 TokenKind
416 Scanner::next()
417 {
418 assert(!pushed_);
420 for (;;) {
421 begin_ = cur_;
423 char c = getChar();
424 switch (c) {
425 case '/':
426 {
427 c = getChar();
428 if (c == '/') {
429 eatSingleLine();
430 break;
431 }
432 if (c == '*') {
433 if (!eatComment()) {
434 cc_.reportError(begin(), Message_UnterminatedComment);
435 return TOK_ERROR;
436 }
437 break;
438 }
439 putChar(c);
440 return TOK_SLASH;
441 }
443 case '\n':
444 case '\r':
445 calcNewLine(c);
446 break;
448 case ';': return TOK_SEMICOLON;
449 case '{': return TOK_LBRACE;
450 case '}': return TOK_RBRACE;
451 case '(': return TOK_LPAREN;
452 case ')': return TOK_RPAREN;
453 case '[': return TOK_LBRACKET;
454 case ']': return TOK_RBRACKET;
455 case '~': return TOK_TILDE;
456 case '?': return TOK_QMARK;
457 case ':': return TOK_COLON;
459 case '.':
460 c = getChar();
461 if (c == '.') {
462 c = getChar();
463 if (c == '.')
464 return TOK_ELLIPSES;
465 }
466 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
467 return TOK_ERROR;
469 case '*':
470 c = getChar();
471 if (c == '=')
472 return TOK_ASSIGN_MUL;
473 putChar(c);
474 return TOK_STAR;
476 case '+':
477 c = getChar();
478 if (c == '=')
479 return TOK_ASSIGN_ADD;
480 if (c == '+')
481 return TOK_INCREMENT;
482 putChar(c);
483 return TOK_PLUS;
485 case '&':
486 c = getChar();
487 if (c == '=')
488 return TOK_ASSIGN_BITAND;
489 if (c == '&')
490 return TOK_AND;
491 putChar(c);
492 return TOK_BITAND;
494 case '|':
495 c = getChar();
496 if (c == '=')
497 return TOK_ASSIGN_BITOR;
498 if (c == '|')
499 return TOK_OR;
500 putChar(c);
501 return TOK_BITOR;
503 case '^':
504 c = getChar();
505 if (c == '=')
506 return TOK_ASSIGN_BITXOR;
507 putChar(c);
508 return TOK_BITXOR;
510 case '%':
511 c = getChar();
512 if (c == '=')
513 return TOK_ASSIGN_MOD;
514 putChar(c);
515 return TOK_PERCENT;
517 case '-':
518 c = getChar();
519 if (c == '=')
520 return TOK_ASSIGN_SUB;
521 if (c == '-')
522 return TOK_DECREMENT;
523 putChar(c);
524 return TOK_MINUS;
526 case ',': return TOK_COMMA;
527 case '\0': return TOK_EOF;
529 case '!':
530 c = getChar();
531 if (c == '=')
532 return TOK_NOTEQUALS;
533 putChar(c);
534 return TOK_NOT;
536 case '=':
537 c = getChar();
538 if (c == '=')
539 return TOK_EQUALS;
540 putChar(c);
541 return TOK_ASSIGN;
543 case '<':
544 c = getChar();
545 if (c == '=')
546 return TOK_LE;
547 if (c == '<') {
548 c = getChar();
549 if (c == '=')
550 return TOK_ASSIGN_SHL;
551 putChar(c);
552 return TOK_SHL;
553 }
554 putChar(c);
555 return TOK_LT;
557 case '>':
558 c = getChar();
559 if (c == '=')
560 return TOK_GE;
561 if (c == '>') {
562 c = getChar();
563 if (c == '>') {
564 c = getChar();
565 if (c == '=')
566 return TOK_ASSIGN_USHR;
567 putChar(c);
568 return TOK_USHR;
569 }
570 return TOK_SHR;
571 }
572 putChar(c);
573 return TOK_GT;
575 case ' ':
576 case '\t':
577 case '\f':
578 break;
580 case '"':
581 return stringLiteral();
583 default:
584 if (c >= '0' && c <= '9')
585 return numberLiteral(c);
586 if (c == '_' || (isascii(c) && isalpha(c)))
587 return identifier(c);
588 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
589 return TOK_ERROR;
590 }
591 }
592 }
594 TokenKind
595 Scanner::scan()
596 {
597 if (pushed_) {
598 pushed_ = false;
599 return lastToken_;
600 }
601 lastToken_ = next();
602 return lastToken_;
603 }
605 TokenKind
606 Scanner::peek()
607 {
608 if (pushed_)
609 return lastToken_;
610 scan();
611 pushed_ = true;
612 return lastToken_;
613 }