| author | David Anderson <dvander@alliedmods.net> |
| Sat Aug 04 19:33:06 2012 -0700 (2012-08-04) | |
| changeset 147 | 9c544823d090 |
| parent 146 | dc7d3b7a918c |
| child 148 | c0740249963c |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "Scanner.h"
25 #include "Utility.h"
27 using namespace ke;
29 const char *ke::TokenNames[] =
30 {
31 #define _(name, str) str,
32 TOKENMAP(_)
33 #undef _
34 NULL
35 };
37 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
38 : cc_(cc),
39 stream_(stream),
40 length_(length),
41 pushed_(false),
42 saved_('\0'),
43 pos_(stream_)
44 {
45 cur_.line = 1;
46 cur_.col = 1;
47 last_ = cur_;
48 }
50 char
51 Scanner::getChar()
52 {
53 if (saved_ != '\0') {
54 char temp = saved_;
55 saved_ = '\0';
56 return temp;
57 }
58 if (pos_ >= stream_ + length_)
59 return '\0';
60 last_.col = cur_.col++;
61 return *pos_++;
62 }
64 void
65 Scanner::putChar(char c)
66 {
67 assert(saved_ == '\0');
68 saved_ = c;
69 cur_.col = last_.col;
70 }
72 void
73 Scanner::calcNewLine(char c)
74 {
75 cur_.line++;
76 cur_.col = 1;
77 last_ = cur_;
79 char o;
80 if (c == '\r' && ((o = getChar()) != '\n'))
81 putChar(o);
82 }
84 void
85 Scanner::eatSingleLine()
86 {
87 char c;
88 while ((c = getChar()) != '\0') {
89 if (c == '\n' || c == '\r') {
90 calcNewLine(c);
91 break;
92 }
93 }
94 }
96 bool
97 Scanner::eatComment()
98 {
99 char c;
101 while ((c = getChar()) != '\0') {
102 while (c == '*') {
103 c = getChar();
104 if (c == '\0')
105 return false;
106 if (c == '/')
107 return true;
108 }
109 if (c == '\r' || c == '\n')
110 calcNewLine(c);
111 }
113 return false;
114 }
116 static inline bool
117 IsDigit(char c)
118 {
119 return c >= '0' && c <= '9';
120 }
122 // Based off the logic in sc2.c's ftoi()...
123 double
124 Scanner::ParseDouble(const char *string)
125 {
126 const char *ptr = string;
128 double number = 0.0;
129 while (IsDigit(*ptr)) {
130 number = (number * 10) + (*ptr - '0');
131 ptr++;
132 }
134 assert(*ptr == '.');
135 ptr++;
137 double fraction = 0.0;
138 double multiplier = 1.0;
139 while (IsDigit(*ptr)) {
140 fraction = (fraction * 10) + (*ptr - '0');
141 multiplier = multiplier / 10.0;
142 ptr++;
143 }
145 number += fraction * multiplier;
147 if (*ptr++ == 'e') {
148 int sign = 1;
149 if (*ptr == '-') {
150 sign = -1;
151 ptr++;
152 }
154 int exponent = 0;
155 while (IsDigit(*ptr)) {
156 exponent = (exponent * 10) + (*ptr - '0');
157 ptr++;
158 }
160 multiplier = pow(10.0, exponent * sign);
161 number *= multiplier;
162 }
164 return number;
165 }
167 static inline bool
168 IsHexDigit(char c)
169 {
170 return IsDigit(c) ||
171 (c >= 'a' && c <= 'f') ||
172 (c >= 'A' && c <= 'F');
173 }
175 // Based on the logic for litchar() in sc2.c.
176 int
177 Scanner::readEscapeCode()
178 {
179 char c = getChar();
180 if (c == '\\')
181 return c;
183 switch (c) {
184 case 'a':
185 return '\a';
186 case 'b':
187 return '\b';
188 case 'e':
189 return 27; // Apparently \e is non-standard.
190 case 'f':
191 return '\f';
192 case 'n':
193 return '\n';
194 case 'r':
195 return '\r';
196 case 't':
197 return '\t';
198 case '\v':
199 return '\v';
201 case 'x':
202 {
203 unsigned digits = 0;
204 char r = 0;
206 c = getChar();
207 while (IsHexDigit(c) && digits < 2) {
208 if (IsDigit(c))
209 c = (c << 4) + (c - '0');
210 else
211 c = (c << 4) + (tolower(c) - 'a' + 10);
212 digits++;
213 c = getChar();
214 }
216 // Swallow a trailing ';'
217 if (c != ';')
218 putChar(c);
220 return r;
221 }
223 case '\'':
224 case '\"':
225 case '%':
226 return c;
228 default:
229 {
230 if (IsDigit(c)) {
231 // \ddd
232 char r = 0;
233 while (IsDigit(c)) {
234 r = r * 10 + (c - '0');
235 c = getChar();
236 }
238 // Swallow a trailing ;
239 if (c != ';')
240 putChar(c);
242 return r;
243 }
244 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
245 return INT_MAX;
246 }
247 }
248 }
250 TokenKind
251 Scanner::stringLiteral()
252 {
253 literal_.clear();
255 for (;;) {
256 char c = getChar();
257 if (c == '\"')
258 break;
259 if (c == '\r' || c == '\n' || c == '\0') {
260 cc_.reportError(begin(), Message_UnterminatedString);
261 return TOK_ERROR;
262 }
263 if (c == '\\') {
264 int code = readEscapeCode();
265 if (code == INT_MAX)
266 return TOK_ERROR;
267 c = char(code);
268 }
269 if (!literal_.append(c))
270 return TOK_ERROR;
271 }
273 if (!literal_.append('\0'))
274 return TOK_ERROR;
276 return TOK_STRING_LITERAL;
277 }
279 TokenKind
280 Scanner::numberLiteral(char first)
281 {
282 literal_.clear();
283 if (!literal_.append(first))
284 return TOK_ERROR;
286 char c;
287 for (;;) {
288 c = getChar();
289 if (!IsDigit(c))
290 break;
291 if (!literal_.append(c))
292 return TOK_ERROR;
293 }
295 if (c != '.') {
296 putChar(c);
297 if (!literal_.append('\0'))
298 return TOK_ERROR;
299 return TOK_INTEGER_LITERAL;
300 }
301 if (!literal_.append(c))
302 return TOK_ERROR;
304 c = getChar();
305 if (!IsDigit(c)) {
306 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
307 return TOK_ERROR;
308 }
309 if (!literal_.append(c))
310 return TOK_ERROR;
312 for (;;) {
313 c = getChar();
314 if (!IsDigit(c))
315 break;
316 if (!literal_.append(c))
317 return TOK_ERROR;
318 }
320 if (c != 'e') {
321 putChar(c);
322 } else {
323 if (!literal_.append(c))
324 return TOK_ERROR;
325 c = getChar();
326 if (c == '-') {
327 if (!literal_.append(c))
328 return TOK_ERROR;
329 c = getChar();
330 }
331 if (!IsDigit(c)) {
332 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
333 return TOK_ERROR;
334 }
335 if (!literal_.append(c))
336 return TOK_ERROR;
337 for (;;) {
338 c = getChar();
339 if (!IsDigit(c)) {
340 putChar(c);
341 break;
342 }
343 if (!literal_.append(c))
344 return TOK_ERROR;
345 }
346 }
348 if (!literal_.append('\0'))
349 return TOK_ERROR;
351 return TOK_FLOAT_LITERAL;
352 }
354 TokenKind
355 Scanner::identifier(char first)
356 {
357 literal_.clear();
358 if (!literal_.append(first))
359 return TOK_ERROR;
361 char c;
362 for (;;) {
363 c = getChar();
364 if (!isascii(c) || (!isalnum(c) && c != '_')) {
365 if (c != ':')
366 putChar(c);
367 break;
368 }
369 if (!literal_.append(c))
370 return TOK_ERROR;
371 }
373 if (!literal_.append('\0'))
374 return TOK_ERROR;
376 if (c != ':' && !strcmp(literal(), "_"))
377 return TOK_UNDERSCORE;
378 if (!strcmp(literal(), "new"))
379 return TOK_NEW;
380 if (!strcmp(literal(), "for"))
381 return TOK_FOR;
382 if (!strcmp(literal(), "return"))
383 return TOK_RETURN;
384 if (!strcmp(literal(), "if"))
385 return TOK_IF;
386 if (!strcmp(literal(), "else"))
387 return TOK_ELSE;
388 if (!strcmp(literal(), "enum"))
389 return TOK_ENUM;
390 if (!strcmp(literal(), "while"))
391 return TOK_WHILE;
392 if (!strcmp(literal(), "do"))
393 return TOK_DO;
394 if (!strcmp(literal(), "break"))
395 return TOK_BREAK;
396 if (!strcmp(literal(), "continue"))
397 return TOK_CONTINUE;
398 if (!strcmp(literal(), "forward"))
399 return TOK_FORWARD;
400 if (!strcmp(literal(), "public"))
401 return TOK_PUBLIC;
402 if (!strcmp(literal(), "const"))
403 return TOK_CONST;
404 if (!strcmp(literal(), "native"))
405 return TOK_NATIVE;
406 if (!strcmp(literal(), "stock"))
407 return TOK_STOCK;
408 if (!strcmp(literal(), "true"))
409 return TOK_TRUE;
410 if (!strcmp(literal(), "false"))
411 return TOK_FALSE;
412 if (!strcmp(literal(), "struct"))
413 return TOK_STRUCT;
414 return (c == ':') ? TOK_LABEL : TOK_NAME;
415 }
417 TokenKind
418 Scanner::next()
419 {
420 assert(!pushed_);
422 for (;;) {
423 begin_ = cur_;
425 char c = getChar();
426 switch (c) {
427 case '/':
428 {
429 c = getChar();
430 if (c == '/') {
431 eatSingleLine();
432 break;
433 }
434 if (c == '*') {
435 if (!eatComment()) {
436 cc_.reportError(begin(), Message_UnterminatedComment);
437 return TOK_ERROR;
438 }
439 break;
440 }
441 putChar(c);
442 return TOK_SLASH;
443 }
445 case '\n':
446 case '\r':
447 calcNewLine(c);
448 break;
450 case ';': return TOK_SEMICOLON;
451 case '{': return TOK_LBRACE;
452 case '}': return TOK_RBRACE;
453 case '(': return TOK_LPAREN;
454 case ')': return TOK_RPAREN;
455 case '[': return TOK_LBRACKET;
456 case ']': return TOK_RBRACKET;
457 case '~': return TOK_TILDE;
458 case '?': return TOK_QMARK;
459 case ':': return TOK_COLON;
461 case '.':
462 c = getChar();
463 if (c == '.') {
464 c = getChar();
465 if (c == '.')
466 return TOK_ELLIPSES;
467 }
468 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
469 return TOK_ERROR;
471 case '*':
472 c = getChar();
473 if (c == '=')
474 return TOK_ASSIGN_MUL;
475 putChar(c);
476 return TOK_STAR;
478 case '+':
479 c = getChar();
480 if (c == '=')
481 return TOK_ASSIGN_ADD;
482 if (c == '+')
483 return TOK_INCREMENT;
484 putChar(c);
485 return TOK_PLUS;
487 case '&':
488 c = getChar();
489 if (c == '=')
490 return TOK_ASSIGN_BITAND;
491 if (c == '&')
492 return TOK_AND;
493 putChar(c);
494 return TOK_BITAND;
496 case '|':
497 c = getChar();
498 if (c == '=')
499 return TOK_ASSIGN_BITOR;
500 if (c == '|')
501 return TOK_OR;
502 putChar(c);
503 return TOK_BITOR;
505 case '^':
506 c = getChar();
507 if (c == '=')
508 return TOK_ASSIGN_BITXOR;
509 putChar(c);
510 return TOK_BITXOR;
512 case '%':
513 c = getChar();
514 if (c == '=')
515 return TOK_ASSIGN_MOD;
516 putChar(c);
517 return TOK_PERCENT;
519 case '-':
520 c = getChar();
521 if (c == '=')
522 return TOK_ASSIGN_SUB;
523 if (c == '-')
524 return TOK_DECREMENT;
525 putChar(c);
526 return TOK_MINUS;
528 case ',': return TOK_COMMA;
529 case '\0': return TOK_EOF;
531 case '!':
532 c = getChar();
533 if (c == '=')
534 return TOK_NOTEQUALS;
535 putChar(c);
536 return TOK_NOT;
538 case '=':
539 c = getChar();
540 if (c == '=')
541 return TOK_EQUALS;
542 putChar(c);
543 return TOK_ASSIGN;
545 case '<':
546 c = getChar();
547 if (c == '=')
548 return TOK_LE;
549 if (c == '<') {
550 c = getChar();
551 if (c == '=')
552 return TOK_ASSIGN_SHL;
553 putChar(c);
554 return TOK_SHL;
555 }
556 putChar(c);
557 return TOK_LT;
559 case '>':
560 c = getChar();
561 if (c == '=')
562 return TOK_GE;
563 if (c == '>') {
564 c = getChar();
565 if (c == '>') {
566 c = getChar();
567 if (c == '=')
568 return TOK_ASSIGN_USHR;
569 putChar(c);
570 return TOK_USHR;
571 }
572 return TOK_SHR;
573 }
574 putChar(c);
575 return TOK_GT;
577 case ' ':
578 case '\t':
579 case '\f':
580 break;
582 case '"':
583 return stringLiteral();
585 default:
586 if (c >= '0' && c <= '9')
587 return numberLiteral(c);
588 if (c == '_' || (isascii(c) && isalpha(c)))
589 return identifier(c);
590 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
591 return TOK_ERROR;
592 }
593 }
594 }
596 TokenKind
597 Scanner::scan()
598 {
599 if (pushed_) {
600 pushed_ = false;
601 return lastToken_;
602 }
603 lastToken_ = next();
604 return lastToken_;
605 }
607 TokenKind
608 Scanner::peek()
609 {
610 if (pushed_)
611 return lastToken_;
612 scan();
613 pushed_ = true;
614 return lastToken_;
615 }