| author | David Anderson <danderson@mozilla.com> |
| Wed Jul 25 01:30:33 2012 -0700 (2012-07-25) | |
| changeset 92 | e3974e7360e9 |
| parent 86 | 32ad0523b309 |
| child 133 | 7797936ba34c |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "Scanner.h"
25 #include "Utility.h"
27 using namespace ke;
29 const char *ke::TokenNames[] =
30 {
31 #define _(name, str) str,
32 TOKENMAP(_)
33 #undef _
34 NULL
35 };
37 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
38 : cc_(cc),
39 stream_(stream),
40 length_(length),
41 pushed_(false),
42 saved_('\0'),
43 pos_(stream_)
44 {
45 cur_.line = 1;
46 cur_.col = 1;
47 last_ = cur_;
48 }
50 char
51 Scanner::getChar()
52 {
53 if (saved_ != '\0') {
54 char temp = saved_;
55 saved_ = '\0';
56 return temp;
57 }
58 if (pos_ >= stream_ + length_)
59 return '\0';
60 last_.col = cur_.col++;
61 return *pos_++;
62 }
64 void
65 Scanner::putChar(char c)
66 {
67 assert(saved_ == '\0');
68 saved_ = c;
69 cur_.col = last_.col;
70 }
72 void
73 Scanner::calcNewLine(char c)
74 {
75 cur_.line++;
76 cur_.col = 1;
77 last_ = cur_;
79 char o;
80 if (c == '\r' && ((o = getChar()) != '\n'))
81 putChar(o);
82 }
84 void
85 Scanner::eatSingleLine()
86 {
87 char c;
88 while ((c = getChar()) != '\0') {
89 if (c == '\n' || c == '\r') {
90 calcNewLine(c);
91 break;
92 }
93 }
94 }
96 bool
97 Scanner::eatComment()
98 {
99 char c;
101 while ((c = getChar()) != '\0') {
102 while (c == '*') {
103 c = getChar();
104 if (c == '\0')
105 return false;
106 if (c == '/')
107 return true;
108 }
109 if (c == '\r' || c == '\n')
110 calcNewLine(c);
111 }
113 return false;
114 }
116 static inline bool
117 IsDigit(char c)
118 {
119 return c >= '0' && c <= '9';
120 }
122 // Based off the logic in sc2.c's ftoi()...
123 double
124 Scanner::ParseDouble(const char *string)
125 {
126 const char *ptr = string;
128 double number = 0.0;
129 while (IsDigit(*ptr)) {
130 number = (number * 10) + (*ptr - '0');
131 ptr++;
132 }
134 assert(*ptr == '.');
135 ptr++;
137 double fraction = 0.0;
138 double multiplier = 1.0;
139 while (IsDigit(*ptr)) {
140 fraction = (fraction * 10) + (*ptr - '0');
141 multiplier = multiplier / 10.0;
142 ptr++;
143 }
145 number += fraction * multiplier;
147 if (*ptr++ == 'e') {
148 int sign = 1;
149 if (*ptr == '-') {
150 sign = -1;
151 ptr++;
152 }
154 int exponent = 0;
155 while (IsDigit(*ptr)) {
156 exponent = (exponent * 10) + (*ptr - '0');
157 ptr++;
158 }
160 multiplier = pow(10.0, exponent * sign);
161 number *= multiplier;
162 }
164 return number;
165 }
167 static inline bool
168 IsHexDigit(char c)
169 {
170 return IsDigit(c) ||
171 (c >= 'a' && c <= 'f') ||
172 (c >= 'A' && c <= 'F');
173 }
175 // Based on the logic for litchar() in sc2.c.
176 int
177 Scanner::readEscapeCode()
178 {
179 char c = getChar();
180 if (c == '\\')
181 return c;
183 switch (c) {
184 case 'a':
185 return '\a';
186 case 'b':
187 return '\b';
188 case 'e':
189 return 27; // Apparently \e is non-standard.
190 case 'f':
191 return '\f';
192 case 'n':
193 return '\n';
194 case 'r':
195 return '\r';
196 case 't':
197 return '\t';
198 case '\v':
199 return '\v';
201 case 'x':
202 {
203 unsigned digits = 0;
204 char r = 0;
206 c = getChar();
207 while (IsHexDigit(c) && digits < 2) {
208 if (IsDigit(c))
209 c = (c << 4) + (c - '0');
210 else
211 c = (c << 4) + (tolower(c) - 'a' + 10);
212 digits++;
213 c = getChar();
214 }
216 // Swallow a trailing ';'
217 if (c != ';')
218 putChar(c);
220 return r;
221 }
223 case '\'':
224 case '\"':
225 case '%':
226 return c;
228 default:
229 {
230 if (IsDigit(c)) {
231 // \ddd
232 char r = 0;
233 while (IsDigit(c)) {
234 r = r * 10 + (c - '0');
235 c = getChar();
236 }
238 // Swallow a trailing ;
239 if (c != ';')
240 putChar(c);
242 return r;
243 }
244 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
245 return INT_MAX;
246 }
247 }
248 }
250 TokenKind
251 Scanner::stringLiteral()
252 {
253 literal_.clear();
255 for (;;) {
256 char c = getChar();
257 if (c == '\"')
258 break;
259 if (c == '\r' || c == '\n' || c == '\0') {
260 cc_.reportError(begin(), Message_UnterminatedString);
261 return TOK_ERROR;
262 }
263 if (c == '\\') {
264 int code = readEscapeCode();
265 if (code == INT_MAX)
266 return TOK_ERROR;
267 c = char(code);
268 }
269 if (!literal_.append(c))
270 return TOK_ERROR;
271 }
273 if (!literal_.append('\0'))
274 return TOK_ERROR;
276 return TOK_STRING_LITERAL;
277 }
279 TokenKind
280 Scanner::numberLiteral(char first)
281 {
282 literal_.clear();
283 if (!literal_.append(first))
284 return TOK_ERROR;
286 char c;
287 for (;;) {
288 c = getChar();
289 if (!IsDigit(c))
290 break;
291 if (!literal_.append(c))
292 return TOK_ERROR;
293 }
295 if (c != '.') {
296 putChar(c);
297 if (!literal_.append('\0'))
298 return TOK_ERROR;
299 return TOK_INTEGER_LITERAL;
300 }
301 if (!literal_.append(c))
302 return TOK_ERROR;
304 c = getChar();
305 if (!IsDigit(c)) {
306 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
307 return TOK_ERROR;
308 }
309 if (!literal_.append(c))
310 return TOK_ERROR;
312 for (;;) {
313 c = getChar();
314 if (!IsDigit(c))
315 break;
316 if (!literal_.append(c))
317 return TOK_ERROR;
318 }
320 if (c != 'e') {
321 putChar(c);
322 } else {
323 if (!literal_.append(c))
324 return TOK_ERROR;
325 c = getChar();
326 if (c == '-') {
327 if (!literal_.append(c))
328 return TOK_ERROR;
329 c = getChar();
330 }
331 if (!IsDigit(c)) {
332 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
333 return TOK_ERROR;
334 }
335 if (!literal_.append(c))
336 return TOK_ERROR;
337 for (;;) {
338 c = getChar();
339 if (!IsDigit(c)) {
340 putChar(c);
341 break;
342 }
343 if (!literal_.append(c))
344 return TOK_ERROR;
345 }
346 }
348 if (!literal_.append('\0'))
349 return TOK_ERROR;
351 return TOK_FLOAT_LITERAL;
352 }
354 TokenKind
355 Scanner::identifier(char first)
356 {
357 literal_.clear();
358 if (!literal_.append(first))
359 return TOK_ERROR;
361 char c;
362 for (;;) {
363 c = getChar();
364 if (!isascii(c) || (!isalnum(c) && c != '_')) {
365 if (c != ':')
366 putChar(c);
367 break;
368 }
369 if (!literal_.append(c))
370 return TOK_ERROR;
371 }
373 if (!literal_.append('\0'))
374 return TOK_ERROR;
376 if (!strcmp(literal(), "new"))
377 return TOK_NEW;
378 if (!strcmp(literal(), "for"))
379 return TOK_FOR;
380 if (!strcmp(literal(), "return"))
381 return TOK_RETURN;
382 if (!strcmp(literal(), "if"))
383 return TOK_IF;
384 if (!strcmp(literal(), "else"))
385 return TOK_ELSE;
386 if (!strcmp(literal(), "enum"))
387 return TOK_ENUM;
388 if (!strcmp(literal(), "while"))
389 return TOK_WHILE;
390 if (!strcmp(literal(), "do"))
391 return TOK_DO;
392 if (!strcmp(literal(), "break"))
393 return TOK_BREAK;
394 if (!strcmp(literal(), "continue"))
395 return TOK_CONTINUE;
396 if (!strcmp(literal(), "forward"))
397 return TOK_FORWARD;
398 if (!strcmp(literal(), "public"))
399 return TOK_PUBLIC;
400 if (!strcmp(literal(), "const"))
401 return TOK_CONST;
402 if (!strcmp(literal(), "native"))
403 return TOK_NATIVE;
404 return (c == ':') ? TOK_LABEL : TOK_NAME;
405 }
407 TokenKind
408 Scanner::next()
409 {
410 assert(!pushed_);
412 for (;;) {
413 begin_ = cur_;
415 char c = getChar();
416 switch (c) {
417 case '/':
418 {
419 c = getChar();
420 if (c == '/') {
421 eatSingleLine();
422 break;
423 }
424 if (c == '*') {
425 if (!eatComment()) {
426 cc_.reportError(begin(), Message_UnterminatedComment);
427 return TOK_ERROR;
428 }
429 break;
430 }
431 putChar(c);
432 return TOK_SLASH;
433 }
435 case '\n':
436 case '\r':
437 calcNewLine(c);
438 break;
440 case ';': return TOK_SEMICOLON;
441 case '{': return TOK_LBRACE;
442 case '}': return TOK_RBRACE;
443 case '(': return TOK_LPAREN;
444 case ')': return TOK_RPAREN;
445 case '[': return TOK_LBRACKET;
446 case ']': return TOK_RBRACKET;
447 case '~': return TOK_TILDE;
448 case '?': return TOK_QMARK;
449 case ':': return TOK_COLON;
451 case '.':
452 c = getChar();
453 if (c == '.') {
454 c = getChar();
455 if (c == '.')
456 return TOK_ELLIPSES;
457 }
458 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
459 return TOK_ERROR;
461 case '*':
462 c = getChar();
463 if (c == '=')
464 return TOK_ASSIGN_MUL;
465 putChar(c);
466 return TOK_STAR;
468 case '+':
469 c = getChar();
470 if (c == '=')
471 return TOK_ASSIGN_ADD;
472 if (c == '+')
473 return TOK_INCREMENT;
474 putChar(c);
475 return TOK_PLUS;
477 case '&':
478 c = getChar();
479 if (c == '=')
480 return TOK_ASSIGN_BITAND;
481 if (c == '&')
482 return TOK_AND;
483 putChar(c);
484 return TOK_BITAND;
486 case '|':
487 c = getChar();
488 if (c == '=')
489 return TOK_ASSIGN_BITOR;
490 if (c == '|')
491 return TOK_OR;
492 putChar(c);
493 return TOK_BITOR;
495 case '^':
496 c = getChar();
497 if (c == '=')
498 return TOK_ASSIGN_BITXOR;
499 putChar(c);
500 return TOK_BITXOR;
502 case '%':
503 c = getChar();
504 if (c == '=')
505 return TOK_ASSIGN_MOD;
506 putChar(c);
507 return TOK_PERCENT;
509 case '-':
510 c = getChar();
511 if (c == '=')
512 return TOK_ASSIGN_SUB;
513 if (c == '-')
514 return TOK_DECREMENT;
515 putChar(c);
516 return TOK_MINUS;
518 case ',': return TOK_COMMA;
519 case '\0': return TOK_EOF;
521 case '!':
522 c = getChar();
523 if (c == '=')
524 return TOK_NOTEQUALS;
525 putChar(c);
526 return TOK_NOT;
528 case '=':
529 c = getChar();
530 if (c == '=')
531 return TOK_EQUALS;
532 putChar(c);
533 return TOK_ASSIGN;
535 case '<':
536 c = getChar();
537 if (c == '=')
538 return TOK_LE;
539 if (c == '<') {
540 c = getChar();
541 if (c == '=')
542 return TOK_ASSIGN_SHL;
543 return TOK_SHL;
544 }
545 putChar(c);
546 return TOK_LT;
548 case '>':
549 c = getChar();
550 if (c == '=')
551 return TOK_GE;
552 if (c == '>') {
553 c = getChar();
554 if (c == '>') {
555 c = getChar();
556 if (c == '=')
557 return TOK_ASSIGN_USHR;
558 putChar(c);
559 return TOK_USHR;
560 }
561 return TOK_SHR;
562 }
563 putChar(c);
564 return TOK_GT;
566 case ' ':
567 case '\t':
568 case '\f':
569 break;
571 case '"':
572 return stringLiteral();
574 default:
575 if (c >= '0' && c <= '9')
576 return numberLiteral(c);
577 if (c == '_' || (isascii(c) && isalpha(c)))
578 return identifier(c);
579 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
580 return TOK_ERROR;
581 }
582 }
583 }
585 TokenKind
586 Scanner::scan()
587 {
588 if (pushed_) {
589 pushed_ = false;
590 return lastToken_;
591 }
592 lastToken_ = next();
593 return lastToken_;
594 }
596 TokenKind
597 Scanner::peek()
598 {
599 if (pushed_)
600 return lastToken_;
601 scan();
602 pushed_ = true;
603 return lastToken_;
604 }