| author | David Anderson <dvander@alliedmods.net> |
| Sun Jul 29 19:27:26 2012 -0700 (2012-07-29) | |
| changeset 134 | 76dec1323d1f |
| parent 133 | 7797936ba34c |
| child 138 | 85196d4d4181 |
| permissions | -rw-r--r-- |
1 /* vim: set ts=4 sw=4 tw=99 et:
2 *
3 * Copyright (C) 2012 David Anderson
4 *
5 * This file is part of JITCraft.
6 *
7 * JITCraft is free software: you can redistribute it and/or modify it under
8 * the terms of the GNU General Public License as published by the Free
9 * Software Foundation, either version 3 of the License, or (at your option)
10 * any later version.
11 *
12 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
13 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * JITCraft. If not, see http://www.gnu.org/licenses/.
18 */
19 #include <assert.h>
20 #include <ctype.h>
21 #include <math.h>
22 #include <limits.h>
23 #include <string.h>
24 #include "Scanner.h"
25 #include "Utility.h"
27 using namespace ke;
29 const char *ke::TokenNames[] =
30 {
31 #define _(name, str) str,
32 TOKENMAP(_)
33 #undef _
34 NULL
35 };
37 Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
38 : cc_(cc),
39 stream_(stream),
40 length_(length),
41 pushed_(false),
42 saved_('\0'),
43 pos_(stream_)
44 {
45 cur_.line = 1;
46 cur_.col = 1;
47 last_ = cur_;
48 }
50 char
51 Scanner::getChar()
52 {
53 if (saved_ != '\0') {
54 char temp = saved_;
55 saved_ = '\0';
56 return temp;
57 }
58 if (pos_ >= stream_ + length_)
59 return '\0';
60 last_.col = cur_.col++;
61 return *pos_++;
62 }
64 void
65 Scanner::putChar(char c)
66 {
67 assert(saved_ == '\0');
68 saved_ = c;
69 cur_.col = last_.col;
70 }
72 void
73 Scanner::calcNewLine(char c)
74 {
75 cur_.line++;
76 cur_.col = 1;
77 last_ = cur_;
79 char o;
80 if (c == '\r' && ((o = getChar()) != '\n'))
81 putChar(o);
82 }
84 void
85 Scanner::eatSingleLine()
86 {
87 char c;
88 while ((c = getChar()) != '\0') {
89 if (c == '\n' || c == '\r') {
90 calcNewLine(c);
91 break;
92 }
93 }
94 }
96 bool
97 Scanner::eatComment()
98 {
99 char c;
101 while ((c = getChar()) != '\0') {
102 while (c == '*') {
103 c = getChar();
104 if (c == '\0')
105 return false;
106 if (c == '/')
107 return true;
108 }
109 if (c == '\r' || c == '\n')
110 calcNewLine(c);
111 }
113 return false;
114 }
116 static inline bool
117 IsDigit(char c)
118 {
119 return c >= '0' && c <= '9';
120 }
122 // Based off the logic in sc2.c's ftoi()...
123 double
124 Scanner::ParseDouble(const char *string)
125 {
126 const char *ptr = string;
128 double number = 0.0;
129 while (IsDigit(*ptr)) {
130 number = (number * 10) + (*ptr - '0');
131 ptr++;
132 }
134 assert(*ptr == '.');
135 ptr++;
137 double fraction = 0.0;
138 double multiplier = 1.0;
139 while (IsDigit(*ptr)) {
140 fraction = (fraction * 10) + (*ptr - '0');
141 multiplier = multiplier / 10.0;
142 ptr++;
143 }
145 number += fraction * multiplier;
147 if (*ptr++ == 'e') {
148 int sign = 1;
149 if (*ptr == '-') {
150 sign = -1;
151 ptr++;
152 }
154 int exponent = 0;
155 while (IsDigit(*ptr)) {
156 exponent = (exponent * 10) + (*ptr - '0');
157 ptr++;
158 }
160 multiplier = pow(10.0, exponent * sign);
161 number *= multiplier;
162 }
164 return number;
165 }
167 static inline bool
168 IsHexDigit(char c)
169 {
170 return IsDigit(c) ||
171 (c >= 'a' && c <= 'f') ||
172 (c >= 'A' && c <= 'F');
173 }
175 // Based on the logic for litchar() in sc2.c.
176 int
177 Scanner::readEscapeCode()
178 {
179 char c = getChar();
180 if (c == '\\')
181 return c;
183 switch (c) {
184 case 'a':
185 return '\a';
186 case 'b':
187 return '\b';
188 case 'e':
189 return 27; // Apparently \e is non-standard.
190 case 'f':
191 return '\f';
192 case 'n':
193 return '\n';
194 case 'r':
195 return '\r';
196 case 't':
197 return '\t';
198 case '\v':
199 return '\v';
201 case 'x':
202 {
203 unsigned digits = 0;
204 char r = 0;
206 c = getChar();
207 while (IsHexDigit(c) && digits < 2) {
208 if (IsDigit(c))
209 c = (c << 4) + (c - '0');
210 else
211 c = (c << 4) + (tolower(c) - 'a' + 10);
212 digits++;
213 c = getChar();
214 }
216 // Swallow a trailing ';'
217 if (c != ';')
218 putChar(c);
220 return r;
221 }
223 case '\'':
224 case '\"':
225 case '%':
226 return c;
228 default:
229 {
230 if (IsDigit(c)) {
231 // \ddd
232 char r = 0;
233 while (IsDigit(c)) {
234 r = r * 10 + (c - '0');
235 c = getChar();
236 }
238 // Swallow a trailing ;
239 if (c != ';')
240 putChar(c);
242 return r;
243 }
244 cc_.reportError(begin(), Message_UnknownEscapeCode, c);
245 return INT_MAX;
246 }
247 }
248 }
250 TokenKind
251 Scanner::stringLiteral()
252 {
253 literal_.clear();
255 for (;;) {
256 char c = getChar();
257 if (c == '\"')
258 break;
259 if (c == '\r' || c == '\n' || c == '\0') {
260 cc_.reportError(begin(), Message_UnterminatedString);
261 return TOK_ERROR;
262 }
263 if (c == '\\') {
264 int code = readEscapeCode();
265 if (code == INT_MAX)
266 return TOK_ERROR;
267 c = char(code);
268 }
269 if (!literal_.append(c))
270 return TOK_ERROR;
271 }
273 if (!literal_.append('\0'))
274 return TOK_ERROR;
276 return TOK_STRING_LITERAL;
277 }
279 TokenKind
280 Scanner::numberLiteral(char first)
281 {
282 literal_.clear();
283 if (!literal_.append(first))
284 return TOK_ERROR;
286 char c;
287 for (;;) {
288 c = getChar();
289 if (!IsDigit(c))
290 break;
291 if (!literal_.append(c))
292 return TOK_ERROR;
293 }
295 if (c != '.') {
296 putChar(c);
297 if (!literal_.append('\0'))
298 return TOK_ERROR;
299 return TOK_INTEGER_LITERAL;
300 }
301 if (!literal_.append(c))
302 return TOK_ERROR;
304 c = getChar();
305 if (!IsDigit(c)) {
306 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
307 return TOK_ERROR;
308 }
309 if (!literal_.append(c))
310 return TOK_ERROR;
312 for (;;) {
313 c = getChar();
314 if (!IsDigit(c))
315 break;
316 if (!literal_.append(c))
317 return TOK_ERROR;
318 }
320 if (c != 'e') {
321 putChar(c);
322 } else {
323 if (!literal_.append(c))
324 return TOK_ERROR;
325 c = getChar();
326 if (c == '-') {
327 if (!literal_.append(c))
328 return TOK_ERROR;
329 c = getChar();
330 }
331 if (!IsDigit(c)) {
332 cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
333 return TOK_ERROR;
334 }
335 if (!literal_.append(c))
336 return TOK_ERROR;
337 for (;;) {
338 c = getChar();
339 if (!IsDigit(c)) {
340 putChar(c);
341 break;
342 }
343 if (!literal_.append(c))
344 return TOK_ERROR;
345 }
346 }
348 if (!literal_.append('\0'))
349 return TOK_ERROR;
351 return TOK_FLOAT_LITERAL;
352 }
354 TokenKind
355 Scanner::identifier(char first)
356 {
357 literal_.clear();
358 if (!literal_.append(first))
359 return TOK_ERROR;
361 char c;
362 for (;;) {
363 c = getChar();
364 if (!isascii(c) || (!isalnum(c) && c != '_')) {
365 if (c != ':')
366 putChar(c);
367 break;
368 }
369 if (!literal_.append(c))
370 return TOK_ERROR;
371 }
373 if (!literal_.append('\0'))
374 return TOK_ERROR;
376 if (!strcmp(literal(), "_"))
377 return TOK_UNDERSCORE;
378 if (!strcmp(literal(), "new"))
379 return TOK_NEW;
380 if (!strcmp(literal(), "for"))
381 return TOK_FOR;
382 if (!strcmp(literal(), "return"))
383 return TOK_RETURN;
384 if (!strcmp(literal(), "if"))
385 return TOK_IF;
386 if (!strcmp(literal(), "else"))
387 return TOK_ELSE;
388 if (!strcmp(literal(), "enum"))
389 return TOK_ENUM;
390 if (!strcmp(literal(), "while"))
391 return TOK_WHILE;
392 if (!strcmp(literal(), "do"))
393 return TOK_DO;
394 if (!strcmp(literal(), "break"))
395 return TOK_BREAK;
396 if (!strcmp(literal(), "continue"))
397 return TOK_CONTINUE;
398 if (!strcmp(literal(), "forward"))
399 return TOK_FORWARD;
400 if (!strcmp(literal(), "public"))
401 return TOK_PUBLIC;
402 if (!strcmp(literal(), "const"))
403 return TOK_CONST;
404 if (!strcmp(literal(), "native"))
405 return TOK_NATIVE;
406 if (!strcmp(literal(), "stock"))
407 return TOK_STOCK;
408 return (c == ':') ? TOK_LABEL : TOK_NAME;
409 }
411 TokenKind
412 Scanner::next()
413 {
414 assert(!pushed_);
416 for (;;) {
417 begin_ = cur_;
419 char c = getChar();
420 switch (c) {
421 case '/':
422 {
423 c = getChar();
424 if (c == '/') {
425 eatSingleLine();
426 break;
427 }
428 if (c == '*') {
429 if (!eatComment()) {
430 cc_.reportError(begin(), Message_UnterminatedComment);
431 return TOK_ERROR;
432 }
433 break;
434 }
435 putChar(c);
436 return TOK_SLASH;
437 }
439 case '\n':
440 case '\r':
441 calcNewLine(c);
442 break;
444 case ';': return TOK_SEMICOLON;
445 case '{': return TOK_LBRACE;
446 case '}': return TOK_RBRACE;
447 case '(': return TOK_LPAREN;
448 case ')': return TOK_RPAREN;
449 case '[': return TOK_LBRACKET;
450 case ']': return TOK_RBRACKET;
451 case '~': return TOK_TILDE;
452 case '?': return TOK_QMARK;
453 case ':': return TOK_COLON;
455 case '.':
456 c = getChar();
457 if (c == '.') {
458 c = getChar();
459 if (c == '.')
460 return TOK_ELLIPSES;
461 }
462 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
463 return TOK_ERROR;
465 case '*':
466 c = getChar();
467 if (c == '=')
468 return TOK_ASSIGN_MUL;
469 putChar(c);
470 return TOK_STAR;
472 case '+':
473 c = getChar();
474 if (c == '=')
475 return TOK_ASSIGN_ADD;
476 if (c == '+')
477 return TOK_INCREMENT;
478 putChar(c);
479 return TOK_PLUS;
481 case '&':
482 c = getChar();
483 if (c == '=')
484 return TOK_ASSIGN_BITAND;
485 if (c == '&')
486 return TOK_AND;
487 putChar(c);
488 return TOK_BITAND;
490 case '|':
491 c = getChar();
492 if (c == '=')
493 return TOK_ASSIGN_BITOR;
494 if (c == '|')
495 return TOK_OR;
496 putChar(c);
497 return TOK_BITOR;
499 case '^':
500 c = getChar();
501 if (c == '=')
502 return TOK_ASSIGN_BITXOR;
503 putChar(c);
504 return TOK_BITXOR;
506 case '%':
507 c = getChar();
508 if (c == '=')
509 return TOK_ASSIGN_MOD;
510 putChar(c);
511 return TOK_PERCENT;
513 case '-':
514 c = getChar();
515 if (c == '=')
516 return TOK_ASSIGN_SUB;
517 if (c == '-')
518 return TOK_DECREMENT;
519 putChar(c);
520 return TOK_MINUS;
522 case ',': return TOK_COMMA;
523 case '\0': return TOK_EOF;
525 case '!':
526 c = getChar();
527 if (c == '=')
528 return TOK_NOTEQUALS;
529 putChar(c);
530 return TOK_NOT;
532 case '=':
533 c = getChar();
534 if (c == '=')
535 return TOK_EQUALS;
536 putChar(c);
537 return TOK_ASSIGN;
539 case '<':
540 c = getChar();
541 if (c == '=')
542 return TOK_LE;
543 if (c == '<') {
544 c = getChar();
545 if (c == '=')
546 return TOK_ASSIGN_SHL;
547 return TOK_SHL;
548 }
549 putChar(c);
550 return TOK_LT;
552 case '>':
553 c = getChar();
554 if (c == '=')
555 return TOK_GE;
556 if (c == '>') {
557 c = getChar();
558 if (c == '>') {
559 c = getChar();
560 if (c == '=')
561 return TOK_ASSIGN_USHR;
562 putChar(c);
563 return TOK_USHR;
564 }
565 return TOK_SHR;
566 }
567 putChar(c);
568 return TOK_GT;
570 case ' ':
571 case '\t':
572 case '\f':
573 break;
575 case '"':
576 return stringLiteral();
578 default:
579 if (c >= '0' && c <= '9')
580 return numberLiteral(c);
581 if (c == '_' || (isascii(c) && isalpha(c)))
582 return identifier(c);
583 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
584 return TOK_ERROR;
585 }
586 }
587 }
589 TokenKind
590 Scanner::scan()
591 {
592 if (pushed_) {
593 pushed_ = false;
594 return lastToken_;
595 }
596 lastToken_ = next();
597 return lastToken_;
598 }
600 TokenKind
601 Scanner::peek()
602 {
603 if (pushed_)
604 return lastToken_;
605 scan();
606 pushed_ = true;
607 return lastToken_;
608 }