| author | David Anderson <dvander@alliedmods.net> |
| Tue Jul 17 03:59:52 2012 -0700 (2012-07-17) | |
| changeset 78 | 93e0a19c805a |
| parent 75 | 662e81d9ed1c |
| child 82 | f99bc938522a |
| permissions | -rw-r--r-- |
1 /*
2 * Copyright (C) 2012 David Anderson
3 *
4 * This file is part of JITCraft.
5 *
6 * JITCraft is free software: you can redistribute it and/or modify it under
7 * the terms of the GNU General Public License as published by the Free
8 * Software Foundation, either version 3 of the License, or (at your option)
9 * any later version.
10 *
11 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * JITCraft. If not, see http://www.gnu.org/licenses/.
17 */
18 #include <assert.h>
19 #include <ctype.h>
20 #include <math.h>
21 #include "Scanner.h"
22 #include "Utility.h"
24 using namespace ke;
26 const char *ke::TokenNames[] =
27 {
28 #define _(name, str) str,
29 TOKENMAP(_)
30 #undef _
31 NULL
32 };
34 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
35 : cc_(cc),
36 stream_(stream),
37 length_(length),
38 pushed_(false),
39 saved_('\0'),
40 pos_(stream_)
41 {
42 cur_.line = 1;
43 cur_.col = 1;
44 last_ = cur_;
45 }
47 char
48 Scanner::getChar()
49 {
50 if (saved_ != '\0') {
51 char temp = saved_;
52 saved_ = '\0';
53 return temp;
54 }
55 if (pos_ >= stream_ + length_)
56 return '\0';
57 last_.col = cur_.col++;
58 return *pos_++;
59 }
61 void
62 Scanner::putChar(char c)
63 {
64 assert(saved_ == '\0');
65 saved_ = c;
66 cur_.col = last_.col;
67 }
69 void
70 Scanner::calcNewLine(char c)
71 {
72 cur_.line++;
73 cur_.col = 1;
74 last_ = cur_;
76 char o;
77 if (c == '\r' && ((o = getChar()) != '\n'))
78 putChar(o);
79 }
81 void
82 Scanner::eatSingleLine()
83 {
84 char c;
85 while ((c = getChar()) != '\0') {
86 if (c == '\n' || c == '\r') {
87 calcNewLine(c);
88 break;
89 }
90 }
91 }
93 bool
94 Scanner::eatComment()
95 {
96 char c;
98 while ((c = getChar()) != '\0') {
99 while (c == '*') {
100 c = getChar();
101 if (c == '\0')
102 return false;
103 if (c == '/')
104 return true;
105 }
106 if (c == '\r' || c == '\n')
107 calcNewLine(c);
108 }
110 return false;
111 }
113 static inline bool
114 IsDigit(char c)
115 {
116 return c >= '0' && c <= '9';
117 }
119 // Based off the logic in sc2.c's ftoi()...
120 double
121 Scanner::ParseDouble(const char *string)
122 {
123 const char *ptr = string;
125 double number = 0.0;
126 while (IsDigit(*ptr)) {
127 number = (number * 10) + (*ptr - '0');
128 ptr++;
129 }
131 assert(*ptr == '.');
132 ptr++;
134 double fraction = 0.0;
135 double multiplier = 1.0;
136 while (IsDigit(*ptr)) {
137 fraction = (fraction * 10) + (*ptr - '0');
138 multiplier = multiplier / 10.0;
139 ptr++;
140 }
142 number += fraction * multiplier;
144 if (*ptr++ == 'e') {
145 int sign = 1;
146 if (*ptr == '-') {
147 sign = -1;
148 ptr++;
149 }
151 int exponent = 0;
152 while (IsDigit(*ptr)) {
153 exponent = (exponent * 10) + (*ptr - '0');
154 ptr++;
155 }
157 multiplier = pow(10.0, exponent * sign);
158 number *= multiplier;
159 }
161 return number;
162 }
164 static inline bool
165 IsHexDigit(char c)
166 {
167 return IsDigit(c) ||
168 (c >= 'a' && c <= 'f') ||
169 (c >= 'A' && c <= 'F');
170 }
172 // Based on the logic for litchar() in sc2.c.
173 int
174 Scanner::readEscapeCode()
175 {
176 char c = getChar();
177 if (c == '\\')
178 return c;
180 switch (c) {
181 case 'a':
182 return '\a';
183 case 'b':
184 return '\b';
185 case 'e':
186 return 27; // Apparently \e is non-standard.
187 case 'f':
188 return '\f';
189 case 'n':
190 return '\n';
191 case 'r':
192 return '\r';
193 case 't':
194 return '\t';
195 case '\v':
196 return '\v';
198 case 'x':
199 {
200 unsigned digits = 0;
201 char r = 0;
203 c = getChar();
204 while (IsHexDigit(c) && digits < 2) {
205 if (IsDigit(c))
206 c = (c << 4) + (c - '0');
207 else
208 c = (c << 4) + (tolower(c) - 'a' + 10);
209 digits++;
210 c = getChar();
211 }
213 // Swallow a trailing ';'
214 if (c != ';')
215 putChar(c);
217 return r;
218 }
220 case '\'':
221 case '\"':
222 case '%':
223 return c;
225 default:
226 {
227 if (IsDigit(c)) {
228 // \ddd
229 char r = 0;
230 while (IsDigit(c)) {
231 r = r * 10 + (c - '0');
232 c = getChar();
233 }
235 // Swallow a trailing ;
236 if (c != ';')
237 putChar(c);
239 return r;
240 }
241 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
242 return INT_MAX;
243 }
244 }
245 }
247 TokenKind
248 Scanner::stringLiteral()
249 {
250 literal_.clear();
252 for (;;) {
253 char c = getChar();
254 if (c == '\"')
255 break;
256 if (c == '\r' || c == '\n' || c == '\0') {
257 cc_.reportError(begin(), Message_UnterminatedString);
258 return TOK_ERROR;
259 }
260 if (c == '\\') {
261 int code = readEscapeCode();
262 if (code == INT_MAX)
263 return TOK_ERROR;
264 c = char(code);
265 }
266 literal_ += c;
267 }
269 return TOK_STRING_LITERAL;
270 }
272 TokenKind
273 Scanner::numberLiteral(char first)
274 {
275 literal_ = first;
277 char c;
278 for (;;) {
279 c = getChar();
280 if (!IsDigit(c))
281 break;
282 literal_ += c;
283 }
285 if (c != '.') {
286 putChar(c);
287 return TOK_INTEGER_LITERAL;
288 }
289 literal_ += c;
291 c = getChar();
292 if (!IsDigit(c)) {
293 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
294 return TOK_ERROR;
295 }
296 literal_ += c;
298 for (;;) {
299 c = getChar();
300 if (!IsDigit(c))
301 break;
302 literal_ += c;
303 }
305 if (c != 'e') {
306 putChar(c);
307 } else {
308 literal_ += c;
309 c = getChar();
310 if (c == '-') {
311 literal_ += c;
312 c = getChar();
313 }
314 if (!IsDigit(c)) {
315 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
316 return TOK_ERROR;
317 }
318 literal_ += c;
319 for (;;) {
320 c = getChar();
321 if (!IsDigit(c)) {
322 putChar(c);
323 break;
324 }
325 literal_ += c;
326 }
327 }
329 return TOK_FLOAT_LITERAL;
330 }
332 TokenKind
333 Scanner::identifier(char first)
334 {
335 literal_ = first;
337 char c;
338 for (;;) {
339 c = getChar();
340 if (!isascii(c) || (!isalnum(c) && c != '_')) {
341 if (c != ':')
342 putChar(c);
343 break;
344 }
345 literal_ += c;
346 }
348 if (literal_ == "new")
349 return TOK_NEW;
350 if (literal_ == "for")
351 return TOK_FOR;
352 if (literal_ == "return")
353 return TOK_RETURN;
354 if (literal_ == "if")
355 return TOK_IF;
356 if (literal_ == "else")
357 return TOK_ELSE;
358 if (literal_ == "enum")
359 return TOK_ENUM;
360 if (literal_ == "while")
361 return TOK_WHILE;
362 if (literal_ == "do")
363 return TOK_DO;
364 if (literal_ == "break")
365 return TOK_BREAK;
366 if (literal_ == "continue")
367 return TOK_CONTINUE;
368 if (literal_ == "forward")
369 return TOK_FORWARD;
370 if (literal_ == "public")
371 return TOK_PUBLIC;
372 if (literal_ == "const")
373 return TOK_CONST;
374 if (literal_ == "native")
375 return TOK_NATIVE;
376 return (c == ':') ? TOK_LABEL : TOK_NAME;
377 }
379 TokenKind
380 Scanner::next()
381 {
382 assert(!pushed_);
384 for (;;) {
385 begin_ = cur_;
387 char c = getChar();
388 switch (c) {
389 case '/':
390 {
391 c = getChar();
392 if (c == '/') {
393 eatSingleLine();
394 break;
395 }
396 if (c == '*') {
397 if (!eatComment()) {
398 cc_.reportError(begin(), Message_UnterminatedComment);
399 return TOK_ERROR;
400 }
401 break;
402 }
403 putChar(c);
404 return TOK_SLASH;
405 }
407 case '\n':
408 case '\r':
409 calcNewLine(c);
410 break;
412 case ';': return TOK_SEMICOLON;
413 case '{': return TOK_LBRACE;
414 case '}': return TOK_RBRACE;
415 case '(': return TOK_LPAREN;
416 case ')': return TOK_RPAREN;
417 case '[': return TOK_LBRACKET;
418 case ']': return TOK_RBRACKET;
419 case '~': return TOK_TILDE;
421 case '.':
422 c = getChar();
423 if (c == '.') {
424 c = getChar();
425 if (c == '.')
426 return TOK_ELLIPSES;
427 }
428 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
429 return TOK_ERROR;
431 case '*':
432 c = getChar();
433 if (c == '=')
434 return TOK_ASSIGN_MUL;
435 putChar(c);
436 return TOK_STAR;
438 case '+':
439 c = getChar();
440 if (c == '=')
441 return TOK_ASSIGN_ADD;
442 if (c == '+')
443 return TOK_INCREMENT;
444 putChar(c);
445 return TOK_PLUS;
447 case '&':
448 c = getChar();
449 if (c == '=')
450 return TOK_ASSIGN_BITAND;
451 if (c == '&')
452 return TOK_AND;
453 putChar(c);
454 return TOK_BITAND;
456 case '|':
457 c = getChar();
458 if (c == '=')
459 return TOK_ASSIGN_BITOR;
460 if (c == '|')
461 return TOK_OR;
462 putChar(c);
463 return TOK_BITOR;
465 case '^':
466 c = getChar();
467 if (c == '=')
468 return TOK_ASSIGN_BITXOR;
469 putChar(c);
470 return TOK_BITXOR;
472 case '-':
473 c = getChar();
474 if (c == '=')
475 return TOK_ASSIGN_SUB;
476 if (c == '-')
477 return TOK_DECREMENT;
478 putChar(c);
479 return TOK_MINUS;
481 case ',': return TOK_COMMA;
482 case '\0': return TOK_EOF;
484 case '!':
485 c = getChar();
486 if (c == '=')
487 return TOK_NOTEQUALS;
488 putChar(c);
489 return TOK_NOT;
491 case '=':
492 c = getChar();
493 if (c == '=')
494 return TOK_EQUALS;
495 putChar(c);
496 return TOK_ASSIGN;
498 case '<':
499 c = getChar();
500 if (c == '=')
501 return TOK_LE;
502 if (c == '<') {
503 c = getChar();
504 if (c == '=')
505 return TOK_ASSIGN_SHL;
506 return TOK_SHL;
507 }
508 putChar(c);
509 return TOK_LT;
511 case '>':
512 c = getChar();
513 if (c == '=')
514 return TOK_GE;
515 if (c == '>') {
516 c = getChar();
517 if (c == '>') {
518 c = getChar();
519 if (c == '=')
520 return TOK_ASSIGN_USHR;
521 putChar(c);
522 return TOK_USHR;
523 }
524 return TOK_SHR;
525 }
526 putChar(c);
527 return TOK_GT;
529 case ' ':
530 case '\t':
531 case '\f':
532 break;
534 case '"':
535 return stringLiteral();
537 default:
538 if (c >= '0' && c <= '9')
539 return numberLiteral(c);
540 if (c == '_' || (isascii(c) && isalpha(c)))
541 return identifier(c);
542 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
543 return TOK_ERROR;
544 }
545 }
546 }
548 TokenKind
549 Scanner::scan()
550 {
551 if (pushed_) {
552 pushed_ = false;
553 return lastToken_;
554 }
555 lastToken_ = next();
556 return lastToken_;
557 }
559 TokenKind
560 Scanner::peek()
561 {
562 if (pushed_)
563 return lastToken_;
564 scan();
565 pushed_ = true;
566 return lastToken_;
567 }