| author | David Anderson <dvander@alliedmods.net> |
| Tue Jul 24 23:53:34 2012 -0700 (2012-07-24) | |
| changeset 86 | 32ad0523b309 |
| parent 83 | jitcraft/Scanner.cpp@79265b56316e |
| child 92 | e3974e7360e9 |
| permissions | -rw-r--r-- |
1 /*
2 * Copyright (C) 2012 David Anderson
3 *
4 * This file is part of JITCraft.
5 *
6 * JITCraft is free software: you can redistribute it and/or modify it under
7 * the terms of the GNU General Public License as published by the Free
8 * Software Foundation, either version 3 of the License, or (at your option)
9 * any later version.
10 *
11 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * JITCraft. If not, see http://www.gnu.org/licenses/.
17 */
18 #include <assert.h>
19 #include <ctype.h>
20 #include <math.h>
21 #include "Scanner.h"
22 #include "Utility.h"
24 using namespace ke;
26 const char *ke::TokenNames[] =
27 {
28 #define _(name, str) str,
29 TOKENMAP(_)
30 #undef _
31 NULL
32 };
34 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
35 : cc_(cc),
36 stream_(stream),
37 length_(length),
38 pushed_(false),
39 saved_('\0'),
40 pos_(stream_)
41 {
42 cur_.line = 1;
43 cur_.col = 1;
44 last_ = cur_;
45 }
47 char
48 Scanner::getChar()
49 {
50 if (saved_ != '\0') {
51 char temp = saved_;
52 saved_ = '\0';
53 return temp;
54 }
55 if (pos_ >= stream_ + length_)
56 return '\0';
57 last_.col = cur_.col++;
58 return *pos_++;
59 }
61 void
62 Scanner::putChar(char c)
63 {
64 assert(saved_ == '\0');
65 saved_ = c;
66 cur_.col = last_.col;
67 }
69 void
70 Scanner::calcNewLine(char c)
71 {
72 cur_.line++;
73 cur_.col = 1;
74 last_ = cur_;
76 char o;
77 if (c == '\r' && ((o = getChar()) != '\n'))
78 putChar(o);
79 }
81 void
82 Scanner::eatSingleLine()
83 {
84 char c;
85 while ((c = getChar()) != '\0') {
86 if (c == '\n' || c == '\r') {
87 calcNewLine(c);
88 break;
89 }
90 }
91 }
93 bool
94 Scanner::eatComment()
95 {
96 char c;
98 while ((c = getChar()) != '\0') {
99 while (c == '*') {
100 c = getChar();
101 if (c == '\0')
102 return false;
103 if (c == '/')
104 return true;
105 }
106 if (c == '\r' || c == '\n')
107 calcNewLine(c);
108 }
110 return false;
111 }
113 static inline bool
114 IsDigit(char c)
115 {
116 return c >= '0' && c <= '9';
117 }
119 // Based off the logic in sc2.c's ftoi()...
120 double
121 Scanner::ParseDouble(const char *string)
122 {
123 const char *ptr = string;
125 double number = 0.0;
126 while (IsDigit(*ptr)) {
127 number = (number * 10) + (*ptr - '0');
128 ptr++;
129 }
131 assert(*ptr == '.');
132 ptr++;
134 double fraction = 0.0;
135 double multiplier = 1.0;
136 while (IsDigit(*ptr)) {
137 fraction = (fraction * 10) + (*ptr - '0');
138 multiplier = multiplier / 10.0;
139 ptr++;
140 }
142 number += fraction * multiplier;
144 if (*ptr++ == 'e') {
145 int sign = 1;
146 if (*ptr == '-') {
147 sign = -1;
148 ptr++;
149 }
151 int exponent = 0;
152 while (IsDigit(*ptr)) {
153 exponent = (exponent * 10) + (*ptr - '0');
154 ptr++;
155 }
157 multiplier = pow(10.0, exponent * sign);
158 number *= multiplier;
159 }
161 return number;
162 }
164 static inline bool
165 IsHexDigit(char c)
166 {
167 return IsDigit(c) ||
168 (c >= 'a' && c <= 'f') ||
169 (c >= 'A' && c <= 'F');
170 }
172 // Based on the logic for litchar() in sc2.c.
173 int
174 Scanner::readEscapeCode()
175 {
176 char c = getChar();
177 if (c == '\\')
178 return c;
180 switch (c) {
181 case 'a':
182 return '\a';
183 case 'b':
184 return '\b';
185 case 'e':
186 return 27; // Apparently \e is non-standard.
187 case 'f':
188 return '\f';
189 case 'n':
190 return '\n';
191 case 'r':
192 return '\r';
193 case 't':
194 return '\t';
195 case '\v':
196 return '\v';
198 case 'x':
199 {
200 unsigned digits = 0;
201 char r = 0;
203 c = getChar();
204 while (IsHexDigit(c) && digits < 2) {
205 if (IsDigit(c))
206 c = (c << 4) + (c - '0');
207 else
208 c = (c << 4) + (tolower(c) - 'a' + 10);
209 digits++;
210 c = getChar();
211 }
213 // Swallow a trailing ';'
214 if (c != ';')
215 putChar(c);
217 return r;
218 }
220 case '\'':
221 case '\"':
222 case '%':
223 return c;
225 default:
226 {
227 if (IsDigit(c)) {
228 // \ddd
229 char r = 0;
230 while (IsDigit(c)) {
231 r = r * 10 + (c - '0');
232 c = getChar();
233 }
235 // Swallow a trailing ;
236 if (c != ';')
237 putChar(c);
239 return r;
240 }
241 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
242 return INT_MAX;
243 }
244 }
245 }
247 TokenKind
248 Scanner::stringLiteral()
249 {
250 literal_.clear();
252 for (;;) {
253 char c = getChar();
254 if (c == '\"')
255 break;
256 if (c == '\r' || c == '\n' || c == '\0') {
257 cc_.reportError(begin(), Message_UnterminatedString);
258 return TOK_ERROR;
259 }
260 if (c == '\\') {
261 int code = readEscapeCode();
262 if (code == INT_MAX)
263 return TOK_ERROR;
264 c = char(code);
265 }
266 literal_ += c;
267 }
269 return TOK_STRING_LITERAL;
270 }
272 TokenKind
273 Scanner::numberLiteral(char first)
274 {
275 literal_ = first;
277 char c;
278 for (;;) {
279 c = getChar();
280 if (!IsDigit(c))
281 break;
282 literal_ += c;
283 }
285 if (c != '.') {
286 putChar(c);
287 return TOK_INTEGER_LITERAL;
288 }
289 literal_ += c;
291 c = getChar();
292 if (!IsDigit(c)) {
293 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
294 return TOK_ERROR;
295 }
296 literal_ += c;
298 for (;;) {
299 c = getChar();
300 if (!IsDigit(c))
301 break;
302 literal_ += c;
303 }
305 if (c != 'e') {
306 putChar(c);
307 } else {
308 literal_ += c;
309 c = getChar();
310 if (c == '-') {
311 literal_ += c;
312 c = getChar();
313 }
314 if (!IsDigit(c)) {
315 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
316 return TOK_ERROR;
317 }
318 literal_ += c;
319 for (;;) {
320 c = getChar();
321 if (!IsDigit(c)) {
322 putChar(c);
323 break;
324 }
325 literal_ += c;
326 }
327 }
329 return TOK_FLOAT_LITERAL;
330 }
332 TokenKind
333 Scanner::identifier(char first)
334 {
335 literal_ = first;
337 char c;
338 for (;;) {
339 c = getChar();
340 if (!isascii(c) || (!isalnum(c) && c != '_')) {
341 if (c != ':')
342 putChar(c);
343 break;
344 }
345 literal_ += c;
346 }
348 if (literal_ == "new")
349 return TOK_NEW;
350 if (literal_ == "for")
351 return TOK_FOR;
352 if (literal_ == "return")
353 return TOK_RETURN;
354 if (literal_ == "if")
355 return TOK_IF;
356 if (literal_ == "else")
357 return TOK_ELSE;
358 if (literal_ == "enum")
359 return TOK_ENUM;
360 if (literal_ == "while")
361 return TOK_WHILE;
362 if (literal_ == "do")
363 return TOK_DO;
364 if (literal_ == "break")
365 return TOK_BREAK;
366 if (literal_ == "continue")
367 return TOK_CONTINUE;
368 if (literal_ == "forward")
369 return TOK_FORWARD;
370 if (literal_ == "public")
371 return TOK_PUBLIC;
372 if (literal_ == "const")
373 return TOK_CONST;
374 if (literal_ == "native")
375 return TOK_NATIVE;
376 return (c == ':') ? TOK_LABEL : TOK_NAME;
377 }
379 TokenKind
380 Scanner::next()
381 {
382 assert(!pushed_);
384 for (;;) {
385 begin_ = cur_;
387 char c = getChar();
388 switch (c) {
389 case '/':
390 {
391 c = getChar();
392 if (c == '/') {
393 eatSingleLine();
394 break;
395 }
396 if (c == '*') {
397 if (!eatComment()) {
398 cc_.reportError(begin(), Message_UnterminatedComment);
399 return TOK_ERROR;
400 }
401 break;
402 }
403 putChar(c);
404 return TOK_SLASH;
405 }
407 case '\n':
408 case '\r':
409 calcNewLine(c);
410 break;
412 case ';': return TOK_SEMICOLON;
413 case '{': return TOK_LBRACE;
414 case '}': return TOK_RBRACE;
415 case '(': return TOK_LPAREN;
416 case ')': return TOK_RPAREN;
417 case '[': return TOK_LBRACKET;
418 case ']': return TOK_RBRACKET;
419 case '~': return TOK_TILDE;
420 case '?': return TOK_QMARK;
421 case ':': return TOK_COLON;
423 case '.':
424 c = getChar();
425 if (c == '.') {
426 c = getChar();
427 if (c == '.')
428 return TOK_ELLIPSES;
429 }
430 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
431 return TOK_ERROR;
433 case '*':
434 c = getChar();
435 if (c == '=')
436 return TOK_ASSIGN_MUL;
437 putChar(c);
438 return TOK_STAR;
440 case '+':
441 c = getChar();
442 if (c == '=')
443 return TOK_ASSIGN_ADD;
444 if (c == '+')
445 return TOK_INCREMENT;
446 putChar(c);
447 return TOK_PLUS;
449 case '&':
450 c = getChar();
451 if (c == '=')
452 return TOK_ASSIGN_BITAND;
453 if (c == '&')
454 return TOK_AND;
455 putChar(c);
456 return TOK_BITAND;
458 case '|':
459 c = getChar();
460 if (c == '=')
461 return TOK_ASSIGN_BITOR;
462 if (c == '|')
463 return TOK_OR;
464 putChar(c);
465 return TOK_BITOR;
467 case '^':
468 c = getChar();
469 if (c == '=')
470 return TOK_ASSIGN_BITXOR;
471 putChar(c);
472 return TOK_BITXOR;
474 case '%':
475 c = getChar();
476 if (c == '=')
477 return TOK_ASSIGN_MOD;
478 putChar(c);
479 return TOK_PERCENT;
481 case '-':
482 c = getChar();
483 if (c == '=')
484 return TOK_ASSIGN_SUB;
485 if (c == '-')
486 return TOK_DECREMENT;
487 putChar(c);
488 return TOK_MINUS;
490 case ',': return TOK_COMMA;
491 case '\0': return TOK_EOF;
493 case '!':
494 c = getChar();
495 if (c == '=')
496 return TOK_NOTEQUALS;
497 putChar(c);
498 return TOK_NOT;
500 case '=':
501 c = getChar();
502 if (c == '=')
503 return TOK_EQUALS;
504 putChar(c);
505 return TOK_ASSIGN;
507 case '<':
508 c = getChar();
509 if (c == '=')
510 return TOK_LE;
511 if (c == '<') {
512 c = getChar();
513 if (c == '=')
514 return TOK_ASSIGN_SHL;
515 return TOK_SHL;
516 }
517 putChar(c);
518 return TOK_LT;
520 case '>':
521 c = getChar();
522 if (c == '=')
523 return TOK_GE;
524 if (c == '>') {
525 c = getChar();
526 if (c == '>') {
527 c = getChar();
528 if (c == '=')
529 return TOK_ASSIGN_USHR;
530 putChar(c);
531 return TOK_USHR;
532 }
533 return TOK_SHR;
534 }
535 putChar(c);
536 return TOK_GT;
538 case ' ':
539 case '\t':
540 case '\f':
541 break;
543 case '"':
544 return stringLiteral();
546 default:
547 if (c >= '0' && c <= '9')
548 return numberLiteral(c);
549 if (c == '_' || (isascii(c) && isalpha(c)))
550 return identifier(c);
551 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
552 return TOK_ERROR;
553 }
554 }
555 }
557 TokenKind
558 Scanner::scan()
559 {
560 if (pushed_) {
561 pushed_ = false;
562 return lastToken_;
563 }
564 lastToken_ = next();
565 return lastToken_;
566 }
568 TokenKind
569 Scanner::peek()
570 {
571 if (pushed_)
572 return lastToken_;
573 scan();
574 pushed_ = true;
575 return lastToken_;
576 }