| author | David Anderson <dvander@alliedmods.net> |
| Sun Jul 29 19:24:55 2012 -0700 (2012-07-29) | |
| changeset 133 | 7797936ba34c |
| parent 92 | e3974e7360e9 |
| child 134 | 76dec1323d1f |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "Scanner.h"
25 #include "Utility.h"
27 using namespace ke;
29 const char *ke::TokenNames[] =
30 {
31 #define _(name, str) str,
32 TOKENMAP(_)
33 #undef _
34 NULL
35 };
37 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
38 : cc_(cc),
39 stream_(stream),
40 length_(length),
41 pushed_(false),
42 saved_('\0'),
43 pos_(stream_)
44 {
45 cur_.line = 1;
46 cur_.col = 1;
47 last_ = cur_;
48 }
50 char
51 Scanner::getChar()
52 {
53 if (saved_ != '\0') {
54 char temp = saved_;
55 saved_ = '\0';
56 return temp;
57 }
58 if (pos_ >= stream_ + length_)
59 return '\0';
60 last_.col = cur_.col++;
61 return *pos_++;
62 }
64 void
65 Scanner::putChar(char c)
66 {
67 assert(saved_ == '\0');
68 saved_ = c;
69 cur_.col = last_.col;
70 }
72 void
73 Scanner::calcNewLine(char c)
74 {
75 cur_.line++;
76 cur_.col = 1;
77 last_ = cur_;
79 char o;
80 if (c == '\r' && ((o = getChar()) != '\n'))
81 putChar(o);
82 }
84 void
85 Scanner::eatSingleLine()
86 {
87 char c;
88 while ((c = getChar()) != '\0') {
89 if (c == '\n' || c == '\r') {
90 calcNewLine(c);
91 break;
92 }
93 }
94 }
96 bool
97 Scanner::eatComment()
98 {
99 char c;
101 while ((c = getChar()) != '\0') {
102 while (c == '*') {
103 c = getChar();
104 if (c == '\0')
105 return false;
106 if (c == '/')
107 return true;
108 }
109 if (c == '\r' || c == '\n')
110 calcNewLine(c);
111 }
113 return false;
114 }
116 static inline bool
117 IsDigit(char c)
118 {
119 return c >= '0' && c <= '9';
120 }
122 // Based off the logic in sc2.c's ftoi()...
123 double
124 Scanner::ParseDouble(const char *string)
125 {
126 const char *ptr = string;
128 double number = 0.0;
129 while (IsDigit(*ptr)) {
130 number = (number * 10) + (*ptr - '0');
131 ptr++;
132 }
134 assert(*ptr == '.');
135 ptr++;
137 double fraction = 0.0;
138 double multiplier = 1.0;
139 while (IsDigit(*ptr)) {
140 fraction = (fraction * 10) + (*ptr - '0');
141 multiplier = multiplier / 10.0;
142 ptr++;
143 }
145 number += fraction * multiplier;
147 if (*ptr++ == 'e') {
148 int sign = 1;
149 if (*ptr == '-') {
150 sign = -1;
151 ptr++;
152 }
154 int exponent = 0;
155 while (IsDigit(*ptr)) {
156 exponent = (exponent * 10) + (*ptr - '0');
157 ptr++;
158 }
160 multiplier = pow(10.0, exponent * sign);
161 number *= multiplier;
162 }
164 return number;
165 }
167 static inline bool
168 IsHexDigit(char c)
169 {
170 return IsDigit(c) ||
171 (c >= 'a' && c <= 'f') ||
172 (c >= 'A' && c <= 'F');
173 }
175 // Based on the logic for litchar() in sc2.c.
176 int
177 Scanner::readEscapeCode()
178 {
179 char c = getChar();
180 if (c == '\\')
181 return c;
183 switch (c) {
184 case 'a':
185 return '\a';
186 case 'b':
187 return '\b';
188 case 'e':
189 return 27; // Apparently \e is non-standard.
190 case 'f':
191 return '\f';
192 case 'n':
193 return '\n';
194 case 'r':
195 return '\r';
196 case 't':
197 return '\t';
198 case '\v':
199 return '\v';
201 case 'x':
202 {
203 unsigned digits = 0;
204 char r = 0;
206 c = getChar();
207 while (IsHexDigit(c) && digits < 2) {
208 if (IsDigit(c))
209 c = (c << 4) + (c - '0');
210 else
211 c = (c << 4) + (tolower(c) - 'a' + 10);
212 digits++;
213 c = getChar();
214 }
216 // Swallow a trailing ';'
217 if (c != ';')
218 putChar(c);
220 return r;
221 }
223 case '\'':
224 case '\"':
225 case '%':
226 return c;
228 default:
229 {
230 if (IsDigit(c)) {
231 // \ddd
232 char r = 0;
233 while (IsDigit(c)) {
234 r = r * 10 + (c - '0');
235 c = getChar();
236 }
238 // Swallow a trailing ;
239 if (c != ';')
240 putChar(c);
242 return r;
243 }
244 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
245 return INT_MAX;
246 }
247 }
248 }
250 TokenKind
251 Scanner::stringLiteral()
252 {
253 literal_.clear();
255 for (;;) {
256 char c = getChar();
257 if (c == '\"')
258 break;
259 if (c == '\r' || c == '\n' || c == '\0') {
260 cc_.reportError(begin(), Message_UnterminatedString);
261 return TOK_ERROR;
262 }
263 if (c == '\\') {
264 int code = readEscapeCode();
265 if (code == INT_MAX)
266 return TOK_ERROR;
267 c = char(code);
268 }
269 if (!literal_.append(c))
270 return TOK_ERROR;
271 }
273 if (!literal_.append('\0'))
274 return TOK_ERROR;
276 return TOK_STRING_LITERAL;
277 }
279 TokenKind
280 Scanner::numberLiteral(char first)
281 {
282 literal_.clear();
283 if (!literal_.append(first))
284 return TOK_ERROR;
286 char c;
287 for (;;) {
288 c = getChar();
289 if (!IsDigit(c))
290 break;
291 if (!literal_.append(c))
292 return TOK_ERROR;
293 }
295 if (c != '.') {
296 putChar(c);
297 if (!literal_.append('\0'))
298 return TOK_ERROR;
299 return TOK_INTEGER_LITERAL;
300 }
301 if (!literal_.append(c))
302 return TOK_ERROR;
304 c = getChar();
305 if (!IsDigit(c)) {
306 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
307 return TOK_ERROR;
308 }
309 if (!literal_.append(c))
310 return TOK_ERROR;
312 for (;;) {
313 c = getChar();
314 if (!IsDigit(c))
315 break;
316 if (!literal_.append(c))
317 return TOK_ERROR;
318 }
320 if (c != 'e') {
321 putChar(c);
322 } else {
323 if (!literal_.append(c))
324 return TOK_ERROR;
325 c = getChar();
326 if (c == '-') {
327 if (!literal_.append(c))
328 return TOK_ERROR;
329 c = getChar();
330 }
331 if (!IsDigit(c)) {
332 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
333 return TOK_ERROR;
334 }
335 if (!literal_.append(c))
336 return TOK_ERROR;
337 for (;;) {
338 c = getChar();
339 if (!IsDigit(c)) {
340 putChar(c);
341 break;
342 }
343 if (!literal_.append(c))
344 return TOK_ERROR;
345 }
346 }
348 if (!literal_.append('\0'))
349 return TOK_ERROR;
351 return TOK_FLOAT_LITERAL;
352 }
354 TokenKind
355 Scanner::identifier(char first)
356 {
357 literal_.clear();
358 if (!literal_.append(first))
359 return TOK_ERROR;
361 char c;
362 for (;;) {
363 c = getChar();
364 if (!isascii(c) || (!isalnum(c) && c != '_')) {
365 if (c != ':')
366 putChar(c);
367 break;
368 }
369 if (!literal_.append(c))
370 return TOK_ERROR;
371 }
373 if (!literal_.append('\0'))
374 return TOK_ERROR;
376 if (!strcmp(literal(), "_"))
377 return TOK_UNDERSCORE;
378 if (!strcmp(literal(), "new"))
379 return TOK_NEW;
380 if (!strcmp(literal(), "for"))
381 return TOK_FOR;
382 if (!strcmp(literal(), "return"))
383 return TOK_RETURN;
384 if (!strcmp(literal(), "if"))
385 return TOK_IF;
386 if (!strcmp(literal(), "else"))
387 return TOK_ELSE;
388 if (!strcmp(literal(), "enum"))
389 return TOK_ENUM;
390 if (!strcmp(literal(), "while"))
391 return TOK_WHILE;
392 if (!strcmp(literal(), "do"))
393 return TOK_DO;
394 if (!strcmp(literal(), "break"))
395 return TOK_BREAK;
396 if (!strcmp(literal(), "continue"))
397 return TOK_CONTINUE;
398 if (!strcmp(literal(), "forward"))
399 return TOK_FORWARD;
400 if (!strcmp(literal(), "public"))
401 return TOK_PUBLIC;
402 if (!strcmp(literal(), "const"))
403 return TOK_CONST;
404 if (!strcmp(literal(), "native"))
405 return TOK_NATIVE;
406 return (c == ':') ? TOK_LABEL : TOK_NAME;
407 }
409 TokenKind
410 Scanner::next()
411 {
412 assert(!pushed_);
414 for (;;) {
415 begin_ = cur_;
417 char c = getChar();
418 switch (c) {
419 case '/':
420 {
421 c = getChar();
422 if (c == '/') {
423 eatSingleLine();
424 break;
425 }
426 if (c == '*') {
427 if (!eatComment()) {
428 cc_.reportError(begin(), Message_UnterminatedComment);
429 return TOK_ERROR;
430 }
431 break;
432 }
433 putChar(c);
434 return TOK_SLASH;
435 }
437 case '\n':
438 case '\r':
439 calcNewLine(c);
440 break;
442 case ';': return TOK_SEMICOLON;
443 case '{': return TOK_LBRACE;
444 case '}': return TOK_RBRACE;
445 case '(': return TOK_LPAREN;
446 case ')': return TOK_RPAREN;
447 case '[': return TOK_LBRACKET;
448 case ']': return TOK_RBRACKET;
449 case '~': return TOK_TILDE;
450 case '?': return TOK_QMARK;
451 case ':': return TOK_COLON;
453 case '.':
454 c = getChar();
455 if (c == '.') {
456 c = getChar();
457 if (c == '.')
458 return TOK_ELLIPSES;
459 }
460 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
461 return TOK_ERROR;
463 case '*':
464 c = getChar();
465 if (c == '=')
466 return TOK_ASSIGN_MUL;
467 putChar(c);
468 return TOK_STAR;
470 case '+':
471 c = getChar();
472 if (c == '=')
473 return TOK_ASSIGN_ADD;
474 if (c == '+')
475 return TOK_INCREMENT;
476 putChar(c);
477 return TOK_PLUS;
479 case '&':
480 c = getChar();
481 if (c == '=')
482 return TOK_ASSIGN_BITAND;
483 if (c == '&')
484 return TOK_AND;
485 putChar(c);
486 return TOK_BITAND;
488 case '|':
489 c = getChar();
490 if (c == '=')
491 return TOK_ASSIGN_BITOR;
492 if (c == '|')
493 return TOK_OR;
494 putChar(c);
495 return TOK_BITOR;
497 case '^':
498 c = getChar();
499 if (c == '=')
500 return TOK_ASSIGN_BITXOR;
501 putChar(c);
502 return TOK_BITXOR;
504 case '%':
505 c = getChar();
506 if (c == '=')
507 return TOK_ASSIGN_MOD;
508 putChar(c);
509 return TOK_PERCENT;
511 case '-':
512 c = getChar();
513 if (c == '=')
514 return TOK_ASSIGN_SUB;
515 if (c == '-')
516 return TOK_DECREMENT;
517 putChar(c);
518 return TOK_MINUS;
520 case ',': return TOK_COMMA;
521 case '\0': return TOK_EOF;
523 case '!':
524 c = getChar();
525 if (c == '=')
526 return TOK_NOTEQUALS;
527 putChar(c);
528 return TOK_NOT;
530 case '=':
531 c = getChar();
532 if (c == '=')
533 return TOK_EQUALS;
534 putChar(c);
535 return TOK_ASSIGN;
537 case '<':
538 c = getChar();
539 if (c == '=')
540 return TOK_LE;
541 if (c == '<') {
542 c = getChar();
543 if (c == '=')
544 return TOK_ASSIGN_SHL;
545 return TOK_SHL;
546 }
547 putChar(c);
548 return TOK_LT;
550 case '>':
551 c = getChar();
552 if (c == '=')
553 return TOK_GE;
554 if (c == '>') {
555 c = getChar();
556 if (c == '>') {
557 c = getChar();
558 if (c == '=')
559 return TOK_ASSIGN_USHR;
560 putChar(c);
561 return TOK_USHR;
562 }
563 return TOK_SHR;
564 }
565 putChar(c);
566 return TOK_GT;
568 case ' ':
569 case '\t':
570 case '\f':
571 break;
573 case '"':
574 return stringLiteral();
576 default:
577 if (c >= '0' && c <= '9')
578 return numberLiteral(c);
579 if (c == '_' || (isascii(c) && isalpha(c)))
580 return identifier(c);
581 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
582 return TOK_ERROR;
583 }
584 }
585 }
587 TokenKind
588 Scanner::scan()
589 {
590 if (pushed_) {
591 pushed_ = false;
592 return lastToken_;
593 }
594 lastToken_ = next();
595 return lastToken_;
596 }
598 TokenKind
599 Scanner::peek()
600 {
601 if (pushed_)
602 return lastToken_;
603 scan();
604 pushed_ = true;
605 return lastToken_;
606 }