| author | David Anderson <dvander@alliedmods.net> |
| Tue Jul 17 00:38:18 2012 -0700 (2012-07-17) | |
| changeset 75 | 662e81d9ed1c |
| parent 74 | 80e29e23eb4e |
| child 78 | 93e0a19c805a |
| permissions | -rw-r--r-- |
1 /*
2 * Copyright (C) 2012 David Anderson
3 *
4 * This file is part of JITCraft.
5 *
6 * JITCraft is free software: you can redistribute it and/or modify it under
7 * the terms of the GNU General Public License as published by the Free
8 * Software Foundation, either version 3 of the License, or (at your option)
9 * any later version.
10 *
11 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * JITCraft. If not, see http://www.gnu.org/licenses/.
17 */
18 #include <assert.h>
19 #include <ctype.h>
20 #include <math.h>
21 #include "Scanner.h"
22 #include "Utility.h"
24 using namespace ke;
26 const char *ke::TokenNames[] =
27 {
28 #define _(name, str) str,
29 TOKENMAP(_)
30 #undef _
31 NULL
32 };
34 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
35 : cc_(cc),
36 stream_(stream),
37 length_(length),
38 pushed_(false),
39 saved_('\0'),
40 pos_(stream_)
41 {
42 cur_.line = 1;
43 cur_.col = 1;
44 last_ = cur_;
45 }
47 char
48 Scanner::getChar()
49 {
50 if (saved_ != '\0') {
51 char temp = saved_;
52 saved_ = '\0';
53 return temp;
54 }
55 if (pos_ >= stream_ + length_)
56 return '\0';
57 last_.col = cur_.col++;
58 return *pos_++;
59 }
61 void
62 Scanner::putChar(char c)
63 {
64 assert(saved_ == '\0');
65 saved_ = c;
66 cur_.col = last_.col;
67 }
69 void
70 Scanner::calcNewLine(char c)
71 {
72 cur_.line++;
73 cur_.col = 1;
74 last_ = cur_;
76 char o;
77 if (c == '\r' && ((o = getChar()) != '\n'))
78 putChar(o);
79 }
81 void
82 Scanner::eatSingleLine()
83 {
84 char c;
85 while ((c = getChar()) != '\0') {
86 if (c == '\n' || c == '\r') {
87 calcNewLine(c);
88 break;
89 }
90 }
91 }
93 bool
94 Scanner::eatComment()
95 {
96 char c;
98 while ((c = getChar()) != '\0') {
99 while (c == '*') {
100 c = getChar();
101 if (c == '\0')
102 return false;
103 if (c == '/')
104 return true;
105 }
106 if (c == '\r' || c == '\n')
107 calcNewLine(c);
108 }
110 return false;
111 }
113 static inline bool
114 IsDigit(char c)
115 {
116 return c >= '0' && c <= '9';
117 }
119 // Based off the logic in sc2.c's ftoi()...
120 double
121 Scanner::ParseDouble(const char *string)
122 {
123 const char *ptr = string;
125 double number = 0.0;
126 while (IsDigit(*ptr)) {
127 number = (number * 10) + (*ptr - '0');
128 ptr++;
129 }
131 assert(*ptr == '.');
132 ptr++;
134 double fraction = 0.0;
135 double multiplier = 1.0;
136 while (IsDigit(*ptr)) {
137 fraction = (fraction * 10) + (*ptr - '0');
138 multiplier = multiplier / 10.0;
139 ptr++;
140 }
142 number += fraction * multiplier;
144 if (*ptr++ == 'e') {
145 int sign = 1;
146 if (*ptr == '-') {
147 sign = -1;
148 ptr++;
149 }
151 int exponent = 0;
152 while (IsDigit(*ptr)) {
153 exponent = (exponent * 10) + (*ptr - '0');
154 ptr++;
155 }
157 multiplier = pow(10.0, exponent * sign);
158 number *= multiplier;
159 }
161 return number;
162 }
164 static inline bool
165 IsHexDigit(char c)
166 {
167 return IsDigit(c) ||
168 (c >= 'a' && c <= 'f') ||
169 (c >= 'A' && c <= 'F');
170 }
172 // Based on the logic for litchar() in sc2.c.
173 int
174 Scanner::readEscapeCode()
175 {
176 char c = getChar();
177 if (c == '\\')
178 return c;
180 switch (c) {
181 case 'a':
182 return '\a';
183 case 'b':
184 return '\b';
185 case 'e':
186 return 27; // Apparently \e is non-standard.
187 case 'f':
188 return '\f';
189 case 'n':
190 return '\n';
191 case 'r':
192 return '\r';
193 case 't':
194 return '\t';
195 case '\v':
196 return '\v';
198 case 'x':
199 {
200 unsigned digits = 0;
201 char r = 0;
203 c = getChar();
204 while (IsHexDigit(c) && digits < 2) {
205 if (IsDigit(c))
206 c = (c << 4) + (c - '0');
207 else
208 c = (c << 4) + (tolower(c) - 'a' + 10);
209 digits++;
210 c = getChar();
211 }
213 // Swallow a trailing ';'
214 if (c != ';')
215 putChar(c);
217 return r;
218 }
220 case '\'':
221 case '\"':
222 case '%':
223 return c;
225 default:
226 {
227 if (IsDigit(c)) {
228 // \ddd
229 char r = 0;
230 while (IsDigit(c)) {
231 r = r * 10 + (c - '0');
232 c = getChar();
233 }
235 // Swallow a trailing ;
236 if (c != ';')
237 putChar(c);
239 return r;
240 }
241 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
242 return INT_MAX;
243 }
244 }
245 }
247 TokenKind
248 Scanner::stringLiteral()
249 {
250 literal_.clear();
252 for (;;) {
253 char c = getChar();
254 if (c == '\"')
255 break;
256 if (c == '\r' || c == '\n' || c == '\0') {
257 cc_.reportError(begin(), Message_UnterminatedString);
258 return TOK_ERROR;
259 }
260 if (c == '\\') {
261 int code = readEscapeCode();
262 if (code == INT_MAX)
263 return TOK_ERROR;
264 c = char(code);
265 }
266 literal_ += c;
267 }
269 return TOK_STRING_LITERAL;
270 }
272 TokenKind
273 Scanner::numberLiteral(char first)
274 {
275 literal_ = first;
277 char c;
278 for (;;) {
279 c = getChar();
280 if (!IsDigit(c))
281 break;
282 literal_ += c;
283 }
285 if (c != '.') {
286 putChar(c);
287 return TOK_INTEGER_LITERAL;
288 }
289 literal_ += c;
291 c = getChar();
292 if (!IsDigit(c)) {
293 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
294 return TOK_ERROR;
295 }
296 literal_ += c;
298 for (;;) {
299 c = getChar();
300 if (!IsDigit(c))
301 break;
302 literal_ += c;
303 }
305 if (c != 'e') {
306 putChar(c);
307 } else {
308 literal_ += c;
309 c = getChar();
310 if (c == '-') {
311 literal_ += c;
312 c = getChar();
313 }
314 if (!IsDigit(c)) {
315 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
316 return TOK_ERROR;
317 }
318 literal_ += c;
319 for (;;) {
320 c = getChar();
321 if (!IsDigit(c)) {
322 putChar(c);
323 break;
324 }
325 literal_ += c;
326 }
327 }
329 return TOK_FLOAT_LITERAL;
330 }
332 TokenKind
333 Scanner::identifier(char first)
334 {
335 literal_ = first;
337 char c;
338 for (;;) {
339 c = getChar();
340 if (!isascii(c) || (!isalnum(c) && c != '_')) {
341 if (c != ':')
342 putChar(c);
343 break;
344 }
345 literal_ += c;
346 }
348 if (literal_ == "new")
349 return TOK_NEW;
350 if (literal_ == "for")
351 return TOK_FOR;
352 if (literal_ == "return")
353 return TOK_RETURN;
354 if (literal_ == "if")
355 return TOK_IF;
356 if (literal_ == "else")
357 return TOK_ELSE;
358 if (literal_ == "enum")
359 return TOK_ENUM;
360 if (literal_ == "while")
361 return TOK_WHILE;
362 if (literal_ == "do")
363 return TOK_DO;
364 if (literal_ == "break")
365 return TOK_BREAK;
366 if (literal_ == "continue")
367 return TOK_CONTINUE;
368 if (literal_ == "forward")
369 return TOK_FORWARD;
370 if (literal_ == "public")
371 return TOK_PUBLIC;
372 if (literal_ == "const")
373 return TOK_CONST;
374 if (literal_ == "native")
375 return TOK_NATIVE;
376 return (c == ':') ? TOK_LABEL : TOK_NAME;
377 }
379 TokenKind
380 Scanner::next()
381 {
382 assert(!pushed_);
384 for (;;) {
385 begin_ = cur_;
387 char c = getChar();
388 switch (c) {
389 case '/':
390 {
391 c = getChar();
392 if (c == '/') {
393 eatSingleLine();
394 break;
395 }
396 if (c == '*') {
397 if (!eatComment()) {
398 cc_.reportError(begin(), Message_UnterminatedComment);
399 return TOK_ERROR;
400 }
401 break;
402 }
403 putChar(c);
404 return TOK_SLASH;
405 }
407 case '\n':
408 case '\r':
409 calcNewLine(c);
410 break;
412 case ';': return TOK_SEMICOLON;
413 case '{': return TOK_LBRACE;
414 case '}': return TOK_RBRACE;
415 case '(': return TOK_LPAREN;
416 case ')': return TOK_RPAREN;
417 case '[': return TOK_LBRACKET;
418 case ']': return TOK_RBRACKET;
420 case '.':
421 c = getChar();
422 if (c == '.') {
423 c = getChar();
424 if (c == '.')
425 return TOK_ELLIPSES;
426 }
427 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
428 return TOK_ERROR;
430 case '*':
431 c = getChar();
432 if (c == '=')
433 return TOK_ASSIGN_MUL;
434 putChar(c);
435 return TOK_STAR;
437 case '+':
438 c = getChar();
439 if (c == '=')
440 return TOK_ASSIGN_ADD;
441 if (c == '+')
442 return TOK_INCREMENT;
443 putChar(c);
444 return TOK_PLUS;
446 case '&':
447 c = getChar();
448 if (c == '=')
449 return TOK_ASSIGN_BITAND;
450 if (c == '&')
451 return TOK_AND;
452 putChar(c);
453 return TOK_BITAND;
455 case '|':
456 c = getChar();
457 if (c == '=')
458 return TOK_ASSIGN_BITOR;
459 if (c == '|')
460 return TOK_OR;
461 putChar(c);
462 return TOK_BITOR;
464 case '^':
465 c = getChar();
466 if (c == '=')
467 return TOK_ASSIGN_BITXOR;
468 putChar(c);
469 return TOK_BITXOR;
471 case '-':
472 c = getChar();
473 if (c == '=')
474 return TOK_ASSIGN_SUB;
475 if (c == '-')
476 return TOK_DECREMENT;
477 putChar(c);
478 return TOK_MINUS;
480 case ',': return TOK_COMMA;
481 case '\0': return TOK_EOF;
483 case '!':
484 c = getChar();
485 if (c == '=')
486 return TOK_NOTEQUALS;
487 putChar(c);
488 return TOK_NOT;
490 case '=':
491 c = getChar();
492 if (c == '=')
493 return TOK_EQUALS;
494 putChar(c);
495 return TOK_ASSIGN;
497 case '<':
498 c = getChar();
499 if (c == '=')
500 return TOK_LE;
501 if (c == '<') {
502 c = getChar();
503 if (c == '=')
504 return TOK_ASSIGN_SHL;
505 return TOK_SHL;
506 }
507 putChar(c);
508 return TOK_LT;
510 case '>':
511 c = getChar();
512 if (c == '=')
513 return TOK_GE;
514 if (c == '>') {
515 c = getChar();
516 if (c == '>') {
517 c = getChar();
518 if (c == '=')
519 return TOK_ASSIGN_USHR;
520 putChar(c);
521 return TOK_USHR;
522 }
523 return TOK_SHR;
524 }
525 putChar(c);
526 return TOK_GT;
528 case ' ':
529 case '\t':
530 case '\f':
531 break;
533 case '"':
534 return stringLiteral();
536 default:
537 if (c >= '0' && c <= '9')
538 return numberLiteral(c);
539 if (c == '_' || (isascii(c) && isalpha(c)))
540 return identifier(c);
541 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
542 return TOK_ERROR;
543 }
544 }
545 }
547 TokenKind
548 Scanner::scan()
549 {
550 if (pushed_) {
551 pushed_ = false;
552 return lastToken_;
553 }
554 lastToken_ = next();
555 return lastToken_;
556 }
558 TokenKind
559 Scanner::peek()
560 {
561 if (pushed_)
562 return lastToken_;
563 scan();
564 pushed_ = true;
565 return lastToken_;
566 }