1.1 --- a/src/Scanner.cpp Mon Sep 17 00:10:01 2012 -0700
1.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000
1.3 @@ -1,679 +0,0 @@
1.4 -/* vim: set ts=4 sw=4 tw=99 et:
1.5 - *
1.6 - * Copyright (C) 2012 David Anderson
1.7 - *
1.8 - * This file is part of JITCraft.
1.9 - *
1.10 - * JITCraft is free software: you can redistribute it and/or modify it under
1.11 - * the terms of the GNU General Public License as published by the Free
1.12 - * Software Foundation, either version 3 of the License, or (at your option)
1.13 - * any later version.
1.14 - *
1.15 - * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
1.16 - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1.17 - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1.18 - *
1.19 - * You should have received a copy of the GNU General Public License along with
1.20 - * JITCraft. If not, see http://www.gnu.org/licenses/.
1.21 - */
1.22 -#include <assert.h>
1.23 -#include <ctype.h>
1.24 -#include <math.h>
1.25 -#include <limits.h>
1.26 -#include <string.h>
1.27 -#include "Scanner.h"
1.28 -#include "Utility.h"
1.29 -
1.30 -using namespace ke;
1.31 -
1.32 -const char *ke::TokenNames[] =
1.33 -{
1.34 -#define _(name, str) str,
1.35 - TOKENMAP(_)
1.36 -#undef _
1.37 - NULL
1.38 -};
1.39 -
1.40 -Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
1.41 - : cc_(cc),
1.42 - stream_(stream),
1.43 - length_(length),
1.44 - pushed_(false),
1.45 - allowTags_(true),
1.46 - saved_('\0'),
1.47 - pos_(stream_)
1.48 -{
1.49 - cur_.line = 1;
1.50 - cur_.col = 1;
1.51 - last_ = cur_;
1.52 -}
1.53 -
1.54 -char
1.55 -Scanner::getChar()
1.56 -{
1.57 - if (saved_ != '\0') {
1.58 - char temp = saved_;
1.59 - saved_ = '\0';
1.60 - return temp;
1.61 - }
1.62 - if (pos_ >= stream_ + length_)
1.63 - return '\0';
1.64 - last_.col = cur_.col++;
1.65 - return *pos_++;
1.66 -}
1.67 -
1.68 -void
1.69 -Scanner::putChar(char c)
1.70 -{
1.71 - assert(saved_ == '\0');
1.72 - saved_ = c;
1.73 - cur_.col = last_.col;
1.74 -}
1.75 -
1.76 -void
1.77 -Scanner::calcNewLine(char c)
1.78 -{
1.79 - cur_.line++;
1.80 - cur_.col = 1;
1.81 - last_ = cur_;
1.82 -
1.83 - char o;
1.84 - if (c == '\r' && ((o = getChar()) != '\n'))
1.85 - putChar(o);
1.86 -}
1.87 -
1.88 -void
1.89 -Scanner::eatSingleLine()
1.90 -{
1.91 - char c;
1.92 - while ((c = getChar()) != '\0') {
1.93 - if (c == '\n' || c == '\r') {
1.94 - calcNewLine(c);
1.95 - break;
1.96 - }
1.97 - }
1.98 -}
1.99 -
1.100 -bool
1.101 -Scanner::eatComment()
1.102 -{
1.103 - char c;
1.104 -
1.105 - while ((c = getChar()) != '\0') {
1.106 - while (c == '*') {
1.107 - c = getChar();
1.108 - if (c == '\0')
1.109 - return false;
1.110 - if (c == '/')
1.111 - return true;
1.112 - }
1.113 - if (c == '\r' || c == '\n')
1.114 - calcNewLine(c);
1.115 - }
1.116 -
1.117 - return false;
1.118 -}
1.119 -
1.120 -static inline bool
1.121 -IsDigit(char c)
1.122 -{
1.123 - return c >= '0' && c <= '9';
1.124 -}
1.125 -
1.126 -// Based off the logic in sc2.c's ftoi()...
1.127 -double
1.128 -Scanner::ParseDouble(const char *string)
1.129 -{
1.130 - const char *ptr = string;
1.131 -
1.132 - double number = 0.0;
1.133 - while (IsDigit(*ptr)) {
1.134 - number = (number * 10) + (*ptr - '0');
1.135 - ptr++;
1.136 - }
1.137 -
1.138 - assert(*ptr == '.');
1.139 - ptr++;
1.140 -
1.141 - double fraction = 0.0;
1.142 - double multiplier = 1.0;
1.143 - while (IsDigit(*ptr)) {
1.144 - fraction = (fraction * 10) + (*ptr - '0');
1.145 - multiplier = multiplier / 10.0;
1.146 - ptr++;
1.147 - }
1.148 -
1.149 - number += fraction * multiplier;
1.150 -
1.151 - if (*ptr++ == 'e') {
1.152 - int sign = 1;
1.153 - if (*ptr == '-') {
1.154 - sign = -1;
1.155 - ptr++;
1.156 - }
1.157 -
1.158 - int exponent = 0;
1.159 - while (IsDigit(*ptr)) {
1.160 - exponent = (exponent * 10) + (*ptr - '0');
1.161 - ptr++;
1.162 - }
1.163 -
1.164 - multiplier = pow(10.0, exponent * sign);
1.165 - number *= multiplier;
1.166 - }
1.167 -
1.168 - return number;
1.169 -}
1.170 -
1.171 -static inline bool
1.172 -IsHexDigit(char c)
1.173 -{
1.174 - return IsDigit(c) ||
1.175 - (c >= 'a' && c <= 'f') ||
1.176 - (c >= 'A' && c <= 'F');
1.177 -}
1.178 -
1.179 -// Based on the logic for litchar() in sc2.c.
1.180 -int
1.181 -Scanner::readEscapeCode()
1.182 -{
1.183 - char c = getChar();
1.184 - if (c == '\\')
1.185 - return c;
1.186 -
1.187 - switch (c) {
1.188 - case 'a':
1.189 - return '\a';
1.190 - case 'b':
1.191 - return '\b';
1.192 - case 'e':
1.193 - return 27; // Apparently \e is non-standard.
1.194 - case 'f':
1.195 - return '\f';
1.196 - case 'n':
1.197 - return '\n';
1.198 - case 'r':
1.199 - return '\r';
1.200 - case 't':
1.201 - return '\t';
1.202 - case '\v':
1.203 - return '\v';
1.204 -
1.205 - case 'x':
1.206 - {
1.207 - unsigned digits = 0;
1.208 - char r = 0;
1.209 -
1.210 - c = getChar();
1.211 - while (IsHexDigit(c) && digits < 2) {
1.212 - if (IsDigit(c))
1.213 - c = (c << 4) + (c - '0');
1.214 - else
1.215 - c = (c << 4) + (tolower(c) - 'a' + 10);
1.216 - digits++;
1.217 - c = getChar();
1.218 - }
1.219 -
1.220 - // Swallow a trailing ';'
1.221 - if (c != ';')
1.222 - putChar(c);
1.223 -
1.224 - return r;
1.225 - }
1.226 -
1.227 - case '\'':
1.228 - case '\"':
1.229 - case '%':
1.230 - return c;
1.231 -
1.232 - default:
1.233 - {
1.234 - if (IsDigit(c)) {
1.235 - // \ddd
1.236 - char r = 0;
1.237 - while (IsDigit(c)) {
1.238 - r = r * 10 + (c - '0');
1.239 - c = getChar();
1.240 - }
1.241 -
1.242 - // Swallow a trailing ;
1.243 - if (c != ';')
1.244 - putChar(c);
1.245 -
1.246 - return r;
1.247 - }
1.248 - cc_.reportError(begin(), Message_UnknownEscapeCode, c);
1.249 - return INT_MAX;
1.250 - }
1.251 - }
1.252 -}
1.253 -
1.254 -TokenKind
1.255 -Scanner::stringLiteral()
1.256 -{
1.257 - literal_.clear();
1.258 -
1.259 - for (;;) {
1.260 - char c = getChar();
1.261 - if (c == '\"')
1.262 - break;
1.263 - if (c == '\r' || c == '\n' || c == '\0') {
1.264 - cc_.reportError(begin(), Message_UnterminatedString);
1.265 - return TOK_ERROR;
1.266 - }
1.267 - if (c == '\\') {
1.268 - int code = readEscapeCode();
1.269 - if (code == INT_MAX)
1.270 - return TOK_ERROR;
1.271 - c = char(code);
1.272 - }
1.273 - if (!literal_.append(c))
1.274 - return TOK_ERROR;
1.275 - }
1.276 -
1.277 - if (!literal_.append('\0'))
1.278 - return TOK_ERROR;
1.279 -
1.280 - return TOK_STRING_LITERAL;
1.281 -}
1.282 -
1.283 -TokenKind
1.284 -Scanner::hexLiteral()
1.285 -{
1.286 - literal_.clear();
1.287 -
1.288 - char c;
1.289 - for (;;) {
1.290 - c = getChar();
1.291 - if (!IsHexDigit(c))
1.292 - break;
1.293 - if (!literal_.append(c))
1.294 - return TOK_ERROR;
1.295 - }
1.296 -
1.297 - putChar(c);
1.298 -
1.299 - if (!literal_.append('\0'))
1.300 - return TOK_ERROR;
1.301 - return TOK_HEX_LITERAL;
1.302 -}
1.303 -
1.304 -TokenKind
1.305 -Scanner::numberLiteral(char first)
1.306 -{
1.307 - literal_.clear();
1.308 - if (!literal_.append(first))
1.309 - return TOK_ERROR;
1.310 -
1.311 - char c;
1.312 - for (;;) {
1.313 - c = getChar();
1.314 - if (!IsDigit(c))
1.315 - break;
1.316 - if (!literal_.append(c))
1.317 - return TOK_ERROR;
1.318 - }
1.319 -
1.320 - // Detect a hexadecimal string.
1.321 - if (literal_.length() == 1 &&
1.322 - literal_[0] == '0' &&
1.323 - (c == 'x' || c == 'X'))
1.324 - {
1.325 - return hexLiteral();
1.326 - }
1.327 -
1.328 - if (c != '.') {
1.329 - putChar(c);
1.330 - if (!literal_.append('\0'))
1.331 - return TOK_ERROR;
1.332 - return TOK_INTEGER_LITERAL;
1.333 - }
1.334 - if (!literal_.append(c))
1.335 - return TOK_ERROR;
1.336 -
1.337 - c = getChar();
1.338 - if (!IsDigit(c)) {
1.339 - cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
1.340 - return TOK_ERROR;
1.341 - }
1.342 - if (!literal_.append(c))
1.343 - return TOK_ERROR;
1.344 -
1.345 - for (;;) {
1.346 - c = getChar();
1.347 - if (!IsDigit(c))
1.348 - break;
1.349 - if (!literal_.append(c))
1.350 - return TOK_ERROR;
1.351 - }
1.352 -
1.353 - if (c != 'e') {
1.354 - putChar(c);
1.355 - } else {
1.356 - if (!literal_.append(c))
1.357 - return TOK_ERROR;
1.358 - c = getChar();
1.359 - if (c == '-') {
1.360 - if (!literal_.append(c))
1.361 - return TOK_ERROR;
1.362 - c = getChar();
1.363 - }
1.364 - if (!IsDigit(c)) {
1.365 - cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
1.366 - return TOK_ERROR;
1.367 - }
1.368 - if (!literal_.append(c))
1.369 - return TOK_ERROR;
1.370 - for (;;) {
1.371 - c = getChar();
1.372 - if (!IsDigit(c)) {
1.373 - putChar(c);
1.374 - break;
1.375 - }
1.376 - if (!literal_.append(c))
1.377 - return TOK_ERROR;
1.378 - }
1.379 - }
1.380 -
1.381 - if (!literal_.append('\0'))
1.382 - return TOK_ERROR;
1.383 -
1.384 - return TOK_FLOAT_LITERAL;
1.385 -}
1.386 -
1.387 -static inline TokenKind
1.388 -FindIdentifier(const char *str)
1.389 -{
1.390 - if (!strcmp(str, "new"))
1.391 - return TOK_NEW;
1.392 - if (!strcmp(str, "for"))
1.393 - return TOK_FOR;
1.394 - if (!strcmp(str, "return"))
1.395 - return TOK_RETURN;
1.396 - if (!strcmp(str, "if"))
1.397 - return TOK_IF;
1.398 - if (!strcmp(str, "else"))
1.399 - return TOK_ELSE;
1.400 - if (!strcmp(str, "enum"))
1.401 - return TOK_ENUM;
1.402 - if (!strcmp(str, "while"))
1.403 - return TOK_WHILE;
1.404 - if (!strcmp(str, "do"))
1.405 - return TOK_DO;
1.406 - if (!strcmp(str, "break"))
1.407 - return TOK_BREAK;
1.408 - if (!strcmp(str, "continue"))
1.409 - return TOK_CONTINUE;
1.410 - if (!strcmp(str, "forward"))
1.411 - return TOK_FORWARD;
1.412 - if (!strcmp(str, "public"))
1.413 - return TOK_PUBLIC;
1.414 - if (!strcmp(str, "const"))
1.415 - return TOK_CONST;
1.416 - if (!strcmp(str, "native"))
1.417 - return TOK_NATIVE;
1.418 - if (!strcmp(str, "stock"))
1.419 - return TOK_STOCK;
1.420 - if (!strcmp(str, "true"))
1.421 - return TOK_TRUE;
1.422 - if (!strcmp(str, "false"))
1.423 - return TOK_FALSE;
1.424 - if (!strcmp(str, "struct"))
1.425 - return TOK_STRUCT;
1.426 - if (!strcmp(str, "switch"))
1.427 - return TOK_SWITCH;
1.428 - if (!strcmp(str, "case"))
1.429 - return TOK_CASE;
1.430 - if (!strcmp(str, "default"))
1.431 - return TOK_DEFAULT;
1.432 - if (!strcmp(str, "typedef"))
1.433 - return TOK_TYPEDEF;
1.434 - if (!strcmp(str, "void"))
1.435 - return TOK_VOID;
1.436 - if (!strcmp(str, "int"))
1.437 - return TOK_INT;
1.438 - if (!strcmp(str, "sizeof"))
1.439 - return TOK_SIZEOF;
1.440 - return TOK_NONE;
1.441 -}
1.442 -
1.443 -TokenKind
1.444 -Scanner::identifier(char first)
1.445 -{
1.446 - literal_.clear();
1.447 - if (!literal_.append(first))
1.448 - return TOK_ERROR;
1.449 -
1.450 - char c;
1.451 - for (;;) {
1.452 - c = getChar();
1.453 - if (!isascii(c) || (!isalnum(c) && c != '_'))
1.454 - break;
1.455 - if (!literal_.append(c))
1.456 - return TOK_ERROR;
1.457 - }
1.458 -
1.459 - if (!literal_.append('\0'))
1.460 - return TOK_ERROR;
1.461 -
1.462 - // Most keywords do not act as labels.
1.463 - TokenKind kind = FindIdentifier(literal());
1.464 - if (kind != TOK_NONE) {
1.465 - putChar(c);
1.466 - return kind;
1.467 - }
1.468 -
1.469 - if (c == ':' && allowTags_)
1.470 - return TOK_LABEL;
1.471 -
1.472 - putChar(c);
1.473 -
1.474 - // _ is a special case which acts as a tag and a keyword.
1.475 - if (!strcmp(literal(), "_"))
1.476 - return TOK_UNDERSCORE;
1.477 -
1.478 - return TOK_NAME;
1.479 -}
1.480 -
1.481 -TokenKind
1.482 -Scanner::next()
1.483 -{
1.484 - assert(!pushed_);
1.485 -
1.486 - for (;;) {
1.487 - begin_ = cur_;
1.488 -
1.489 - char c = getChar();
1.490 - switch (c) {
1.491 - case '/':
1.492 - {
1.493 - c = getChar();
1.494 - if (c == '/') {
1.495 - eatSingleLine();
1.496 - break;
1.497 - }
1.498 - if (c == '*') {
1.499 - if (!eatComment()) {
1.500 - cc_.reportError(begin(), Message_UnterminatedComment);
1.501 - return TOK_ERROR;
1.502 - }
1.503 - break;
1.504 - }
1.505 - putChar(c);
1.506 - return TOK_SLASH;
1.507 - }
1.508 -
1.509 - case '\n':
1.510 - case '\r':
1.511 - calcNewLine(c);
1.512 - break;
1.513 -
1.514 - case ';': return TOK_SEMICOLON;
1.515 - case '{': return TOK_LBRACE;
1.516 - case '}': return TOK_RBRACE;
1.517 - case '(': return TOK_LPAREN;
1.518 - case ')': return TOK_RPAREN;
1.519 - case '[': return TOK_LBRACKET;
1.520 - case ']': return TOK_RBRACKET;
1.521 - case '~': return TOK_TILDE;
1.522 - case '?': return TOK_QMARK;
1.523 - case ':': return TOK_COLON;
1.524 -
1.525 - case '.':
1.526 - c = getChar();
1.527 - if (c == '.') {
1.528 - c = getChar();
1.529 - if (c == '.')
1.530 - return TOK_ELLIPSES;
1.531 - cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
1.532 - return TOK_ERROR;
1.533 - }
1.534 - putChar(c);
1.535 - return TOK_DOT;
1.536 -
1.537 - case '*':
1.538 - c = getChar();
1.539 - if (c == '=')
1.540 - return TOK_ASSIGN_MUL;
1.541 - putChar(c);
1.542 - return TOK_STAR;
1.543 -
1.544 - case '+':
1.545 - c = getChar();
1.546 - if (c == '=')
1.547 - return TOK_ASSIGN_ADD;
1.548 - if (c == '+')
1.549 - return TOK_INCREMENT;
1.550 - putChar(c);
1.551 - return TOK_PLUS;
1.552 -
1.553 - case '&':
1.554 - c = getChar();
1.555 - if (c == '=')
1.556 - return TOK_ASSIGN_BITAND;
1.557 - if (c == '&')
1.558 - return TOK_AND;
1.559 - putChar(c);
1.560 - return TOK_BITAND;
1.561 -
1.562 - case '|':
1.563 - c = getChar();
1.564 - if (c == '=')
1.565 - return TOK_ASSIGN_BITOR;
1.566 - if (c == '|')
1.567 - return TOK_OR;
1.568 - putChar(c);
1.569 - return TOK_BITOR;
1.570 -
1.571 - case '^':
1.572 - c = getChar();
1.573 - if (c == '=')
1.574 - return TOK_ASSIGN_BITXOR;
1.575 - putChar(c);
1.576 - return TOK_BITXOR;
1.577 -
1.578 - case '%':
1.579 - c = getChar();
1.580 - if (c == '=')
1.581 - return TOK_ASSIGN_MOD;
1.582 - putChar(c);
1.583 - return TOK_PERCENT;
1.584 -
1.585 - case '-':
1.586 - c = getChar();
1.587 - if (c == '=')
1.588 - return TOK_ASSIGN_SUB;
1.589 - if (c == '-')
1.590 - return TOK_DECREMENT;
1.591 - putChar(c);
1.592 - return TOK_MINUS;
1.593 -
1.594 - case ',': return TOK_COMMA;
1.595 - case '\0': return TOK_EOF;
1.596 -
1.597 - case '!':
1.598 - c = getChar();
1.599 - if (c == '=')
1.600 - return TOK_NOTEQUALS;
1.601 - putChar(c);
1.602 - return TOK_NOT;
1.603 -
1.604 - case '=':
1.605 - c = getChar();
1.606 - if (c == '=')
1.607 - return TOK_EQUALS;
1.608 - putChar(c);
1.609 - return TOK_ASSIGN;
1.610 -
1.611 -
1.612 - case '<':
1.613 - c = getChar();
1.614 - if (c == '=')
1.615 - return TOK_LE;
1.616 - if (c == '<') {
1.617 - c = getChar();
1.618 - if (c == '=')
1.619 - return TOK_ASSIGN_SHL;
1.620 - putChar(c);
1.621 - return TOK_SHL;
1.622 - }
1.623 - putChar(c);
1.624 - return TOK_LT;
1.625 -
1.626 - case '>':
1.627 - c = getChar();
1.628 - if (c == '=')
1.629 - return TOK_GE;
1.630 - if (c == '>') {
1.631 - c = getChar();
1.632 - if (c == '>') {
1.633 - c = getChar();
1.634 - if (c == '=')
1.635 - return TOK_ASSIGN_USHR;
1.636 - putChar(c);
1.637 - return TOK_USHR;
1.638 - }
1.639 - return TOK_SHR;
1.640 - }
1.641 - putChar(c);
1.642 - return TOK_GT;
1.643 -
1.644 - case ' ':
1.645 - case '\t':
1.646 - case '\f':
1.647 - break;
1.648 -
1.649 - case '"':
1.650 - return stringLiteral();
1.651 -
1.652 - default:
1.653 - if (c >= '0' && c <= '9')
1.654 - return numberLiteral(c);
1.655 - if (c == '_' || (isascii(c) && isalpha(c)))
1.656 - return identifier(c);
1.657 - cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
1.658 - return TOK_ERROR;
1.659 - }
1.660 - }
1.661 -}
1.662 -
1.663 -TokenKind
1.664 -Scanner::scan()
1.665 -{
1.666 - if (pushed_) {
1.667 - pushed_ = false;
1.668 - return lastToken_;
1.669 - }
1.670 - lastToken_ = next();
1.671 - return lastToken_;
1.672 -}
1.673 -
1.674 -TokenKind
1.675 -Scanner::peek()
1.676 -{
1.677 - if (pushed_)
1.678 - return lastToken_;
1.679 - scan();
1.680 - pushed_ = true;
1.681 - return lastToken_;
1.682 -}
