| author | David Anderson <dvander@alliedmods.net> |
| Mon Jul 02 01:24:57 2012 -0700 (2012-07-02) | |
| changeset 25 | 23e8b3d8e4e6 |
| parent 23 | 8a31742781c1 |
| child 28 | a9665857cfd5 |
| permissions | -rw-r--r-- |
1 /*
2 * Copyright (C) 2012 David Anderson
3 *
4 * This file is part of JITCraft.
5 *
6 * JITCraft is free software: you can redistribute it and/or modify it under
7 * the terms of the GNU General Public License as published by the Free
8 * Software Foundation, either version 3 of the License, or (at your option)
9 * any later version.
10 *
11 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
12 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * JITCraft. If not, see http://www.gnu.org/licenses/.
17 */
18 #include <assert.h>
19 #include <ctype.h>
20 #include "Scanner.h"
21 #include "Utility.h"
23 using namespace jc;
25 const char *jc::TokenNames[] =
26 {
27 #define _(name, str) str,
28 TOKENMAP(_)
29 #undef _
30 NULL
31 };
33 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
34 : cc_(cc),
35 stream_(stream),
36 length_(length),
37 pushed_(false),
38 saved_('\0'),
39 pos_(stream_)
40 {
41 cur_.line = 0;
42 cur_.col = 0;
43 last_ = cur_;
44 }
46 char
47 Scanner::getChar()
48 {
49 if (saved_ != '\0') {
50 char temp = saved_;
51 saved_ = '\0';
52 return temp;
53 }
54 if (pos_ >= stream_ + length_)
55 return '\0';
56 last_.col = cur_.col++;
57 return *pos_++;
58 }
60 void
61 Scanner::putChar(char c)
62 {
63 assert(saved_ == '\0');
64 saved_ = c;
65 cur_.col = last_.col;
66 }
68 void
69 Scanner::calcNewLine(char c)
70 {
71 cur_.line++;
72 cur_.col = 0;
73 last_ = cur_;
75 char o;
76 if (c == '\r' && ((o = getChar()) != '\n'))
77 putChar(o);
78 }
80 void
81 Scanner::eatSingleLine()
82 {
83 char c;
84 while ((c = getChar()) != '\0') {
85 if (c == '\n' || c == '\r') {
86 calcNewLine(c);
87 break;
88 }
89 }
90 }
92 bool
93 Scanner::eatComment()
94 {
95 char c;
97 while ((c = getChar()) != '\0') {
98 while (c == '*') {
99 c = getChar();
100 if (c == '\0')
101 return false;
102 if (c == '/')
103 return true;
104 }
105 if (c == '\r' || c == '\n')
106 calcNewLine(c);
107 }
109 return false;
110 }
112 static inline bool
113 IsDigit(char c)
114 {
115 return c >= '0' && c <= '9';
116 }
118 // Based off the logic in sc2.c's ftoi()...
119 double
120 Scanner::ParseDouble(const char *string)
121 {
122 const char *ptr = string;
124 double number = 0.0;
125 while (IsDigit(*ptr)) {
126 number = (number * 10) + (*ptr - '0');
127 ptr++;
128 }
130 assert(*ptr == '.');
131 ptr++;
133 double fraction = 0.0;
134 double multiplier = 1.0;
135 while (IsDigit(*ptr)) {
136 fraction = (fraction * 10) + (*ptr - '0');
137 multiplier = multiplier / 10.0;
138 ptr++;
139 }
141 number += fraction * multiplier;
143 if (*ptr++ == 'e') {
144 int sign = 1;
145 if (*ptr == '-') {
146 sign = -1;
147 ptr++;
148 }
150 int exponent = 0;
151 while (IsDigit(*ptr)) {
152 exponent = (exponent * 10) + (*ptr - '0');
153 ptr++;
154 }
156 multiplier = pow(10.0, exponent * sign);
157 number *= multiplier;
158 }
160 return number;
161 }
163 static inline bool
164 IsHexDigit(char c)
165 {
166 return IsDigit(c) ||
167 (c >= 'a' && c <= 'f') ||
168 (c >= 'A' && c <= 'F');
169 }
171 // Based on the logic for litchar() in sc2.c.
172 int
173 Scanner::readEscapeCode()
174 {
175 char c = getChar();
176 if (c == '\\')
177 return c;
179 switch (c) {
180 case 'a':
181 return '\a';
182 case 'b':
183 return '\b';
184 case 'e':
185 return 27; // Apparently \e is non-standard.
186 case 'f':
187 return '\f';
188 case 'n':
189 return '\n';
190 case 'r':
191 return '\r';
192 case 't':
193 return '\t';
194 case '\v':
195 return '\v';
197 case 'x':
198 {
199 unsigned digits = 0;
200 char r = 0;
202 c = getChar();
203 while (IsHexDigit(c) && digits < 2) {
204 if (IsDigit(c))
205 c = (c << 4) + (c - '0');
206 else
207 c = (c << 4) + (tolower(c) - 'a' + 10);
208 digits++;
209 c = getChar();
210 }
212 // Swallow a trailing ';'
213 if (c != ';')
214 putChar(c);
216 return r;
217 }
219 case '\'':
220 case '\"':
221 case '%':
222 return c;
224 default:
225 {
226 if (IsDigit(c)) {
227 // \ddd
228 char r = 0;
229 while (IsDigit(c)) {
230 r = r * 10 + (c - '0');
231 c = getChar();
232 }
234 // Swallow a trailing ;
235 if (c != ';')
236 putChar(c);
238 return r;
239 }
240 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
241 return INT_MAX;
242 }
243 }
244 }
246 TokenKind
247 Scanner::stringLiteral()
248 {
249 literal_.clear();
251 for (;;) {
252 char c = getChar();
253 if (c == '\"')
254 break;
255 if (c == '\r' || c == '\n' || c == '\0') {
256 cc_.reportError(begin(), Message_UnterminatedString);
257 return TOK_ERROR;
258 }
259 if (c == '\\') {
260 int code = readEscapeCode();
261 if (code == INT_MAX)
262 return TOK_ERROR;
263 c = char(code);
264 }
265 literal_ += c;
266 }
268 return TOK_STRING_LITERAL;
269 }
271 TokenKind
272 Scanner::numberLiteral(char first)
273 {
274 literal_ = first;
276 char c;
277 for (;;) {
278 c = getChar();
279 if (!IsDigit(c))
280 break;
281 literal_ += c;
282 }
284 if (c != '.') {
285 putChar(c);
286 return TOK_INTEGER_LITERAL;
287 }
288 literal_ += c;
290 c = getChar();
291 if (!IsDigit(c)) {
292 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
293 return TOK_ERROR;
294 }
295 literal_ += c;
297 for (;;) {
298 c = getChar();
299 if (!IsDigit(c))
300 break;
301 literal_ += c;
302 }
304 if (c != 'e') {
305 putChar(c);
306 } else {
307 literal_ += c;
308 c = getChar();
309 if (c == '-') {
310 literal_ += c;
311 c = getChar();
312 }
313 if (!IsDigit(c)) {
314 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
315 return TOK_ERROR;
316 }
317 literal_ += c;
318 for (;;) {
319 c = getChar();
320 if (!IsDigit(c)) {
321 putChar(c);
322 break;
323 }
324 literal_ += c;
325 }
326 }
328 return TOK_FLOAT_LITERAL;
329 }
331 TokenKind
332 Scanner::identifier(char first)
333 {
334 literal_ = first;
336 char c;
337 for (;;) {
338 c = getChar();
339 if (!isascii(c) || (!isalnum(c) && c != '_')) {
340 if (c != ':')
341 putChar(c);
342 break;
343 }
344 literal_ += c;
345 }
347 if (literal_ == "new")
348 return TOK_NEW;
349 if (literal_ == "for")
350 return TOK_FOR;
351 if (literal_ == "return")
352 return TOK_RETURN;
353 if (literal_ == "if")
354 return TOK_IF;
355 if (literal_ == "else")
356 return TOK_ELSE;
357 if (literal_ == "enum")
358 return TOK_ENUM;
359 if (literal_ == "while")
360 return TOK_WHILE;
361 if (literal_ == "do")
362 return TOK_DO;
363 if (literal_ == "break")
364 return TOK_BREAK;
365 if (literal_ == "continue")
366 return TOK_CONTINUE;
367 return (c == ':') ? TOK_LABEL : TOK_NAME;
368 }
370 TokenKind
371 Scanner::next()
372 {
373 assert(!pushed_);
375 for (;;) {
376 begin_ = cur_;
378 char c = getChar();
379 switch (c) {
380 case '/':
381 {
382 c = getChar();
383 if (c == '/') {
384 eatSingleLine();
385 break;
386 }
387 if (c == '*') {
388 if (!eatComment()) {
389 cc_.reportError(begin(), Message_UnterminatedComment);
390 return TOK_ERROR;
391 }
392 break;
393 }
394 putChar(c);
395 return TOK_SLASH;
396 }
398 case '\n':
399 case '\r':
400 calcNewLine(c);
401 break;
403 case ';': return TOK_SEMICOLON;
404 case '{': return TOK_LBRACE;
405 case '}': return TOK_RBRACE;
406 case '(': return TOK_LPAREN;
407 case ')': return TOK_RPAREN;
408 case '[': return TOK_LBRACKET;
409 case ']': return TOK_RBRACKET;
411 case '*':
412 c = getChar();
413 if (c == '=')
414 return TOK_ASSIGN_MUL;
415 putChar(c);
416 return TOK_STAR;
418 case '+':
419 c = getChar();
420 if (c == '=')
421 return TOK_ASSIGN_ADD;
422 if (c == '+')
423 return TOK_INCREMENT;
424 putChar(c);
425 return TOK_PLUS;
427 case '&':
428 c = getChar();
429 if (c == '=')
430 return TOK_ASSIGN_BITAND;
431 putChar(c);
432 return TOK_BITAND;
434 case '|':
435 c = getChar();
436 if (c == '=')
437 return TOK_ASSIGN_BITOR;
438 putChar(c);
439 return TOK_BITOR;
441 case '^':
442 c = getChar();
443 if (c == '=')
444 return TOK_ASSIGN_BITXOR;
445 putChar(c);
446 return TOK_BITXOR;
448 case '-':
449 c = getChar();
450 if (c == '=')
451 return TOK_ASSIGN_SUB;
452 if (c == '-')
453 return TOK_DECREMENT;
454 putChar(c);
455 return TOK_MINUS;
457 case ',': return TOK_COMMA;
458 case '\0': return TOK_EOF;
460 case '=':
461 c = getChar();
462 if (c == '=')
463 return TOK_EQUALS;
464 putChar(c);
465 return TOK_ASSIGN;
467 case '<':
468 c = getChar();
469 if (c == '=')
470 return TOK_LE;
471 if (c == '<') {
472 c = getChar();
473 if (c == '=')
474 return TOK_ASSIGN_SHL;
475 return TOK_SHL;
476 }
477 putChar(c);
478 return TOK_LT;
480 case '>':
481 c = getChar();
482 if (c == '=')
483 return TOK_GE;
484 if (c == '>') {
485 c = getChar();
486 if (c == '>') {
487 c = getChar();
488 if (c == '=')
489 return TOK_ASSIGN_USHR;
490 putChar(c);
491 return TOK_USHR;
492 }
493 return TOK_SHR;
494 }
495 putChar(c);
496 return TOK_GT;
498 case ' ':
499 case '\t':
500 case '\f':
501 break;
503 case '"':
504 return stringLiteral();
506 default:
507 if (c >= '0' && c <= '9')
508 return numberLiteral(c);
509 if (c == '_' || (isascii(c) && isalpha(c)))
510 return identifier(c);
511 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
512 return TOK_ERROR;
513 }
514 }
515 }
517 TokenKind
518 Scanner::scan()
519 {
520 if (pushed_) {
521 pushed_ = false;
522 return lastToken_;
523 }
524 lastToken_ = next();
525 return lastToken_;
526 }
528 TokenKind
529 Scanner::peek()
530 {
531 if (pushed_)
532 return lastToken_;
533 scan();
534 pushed_ = true;
535 return lastToken_;
536 }