1.1 --- a/jitcraft/Scanner.cpp Tue Jul 24 23:53:34 2012 -0700
1.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000
1.3 @@ -1,576 +0,0 @@
1.4 -/*
1.5 - * Copyright (C) 2012 David Anderson
1.6 - *
1.7 - * This file is part of JITCraft.
1.8 - *
1.9 - * JITCraft is free software: you can redistribute it and/or modify it under
1.10 - * the terms of the GNU General Public License as published by the Free
1.11 - * Software Foundation, either version 3 of the License, or (at your option)
1.12 - * any later version.
1.13 - *
1.14 - * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
1.15 - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1.16 - * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1.17 - *
1.18 - * You should have received a copy of the GNU General Public License along with
1.19 - * JITCraft. If not, see http://www.gnu.org/licenses/.
1.20 - */
1.21 -#include <assert.h>
1.22 -#include <ctype.h>
1.23 -#include <math.h>
1.24 -#include "Scanner.h"
1.25 -#include "Utility.h"
1.26 -
1.27 -using namespace ke;
1.28 -
1.29 -const char *ke::TokenNames[] =
1.30 -{
1.31 -#define _(name, str) str,
1.32 - TOKENMAP(_)
1.33 -#undef _
1.34 - NULL
1.35 -};
1.36 -
1.37 -Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
1.38 - : cc_(cc),
1.39 - stream_(stream),
1.40 - length_(length),
1.41 - pushed_(false),
1.42 - saved_('\0'),
1.43 - pos_(stream_)
1.44 -{
1.45 - cur_.line = 1;
1.46 - cur_.col = 1;
1.47 - last_ = cur_;
1.48 -}
1.49 -
1.50 -char
1.51 -Scanner::getChar()
1.52 -{
1.53 - if (saved_ != '\0') {
1.54 - char temp = saved_;
1.55 - saved_ = '\0';
1.56 - return temp;
1.57 - }
1.58 - if (pos_ >= stream_ + length_)
1.59 - return '\0';
1.60 - last_.col = cur_.col++;
1.61 - return *pos_++;
1.62 -}
1.63 -
1.64 -void
1.65 -Scanner::putChar(char c)
1.66 -{
1.67 - assert(saved_ == '\0');
1.68 - saved_ = c;
1.69 - cur_.col = last_.col;
1.70 -}
1.71 -
1.72 -void
1.73 -Scanner::calcNewLine(char c)
1.74 -{
1.75 - cur_.line++;
1.76 - cur_.col = 1;
1.77 - last_ = cur_;
1.78 -
1.79 - char o;
1.80 - if (c == '\r' && ((o = getChar()) != '\n'))
1.81 - putChar(o);
1.82 -}
1.83 -
1.84 -void
1.85 -Scanner::eatSingleLine()
1.86 -{
1.87 - char c;
1.88 - while ((c = getChar()) != '\0') {
1.89 - if (c == '\n' || c == '\r') {
1.90 - calcNewLine(c);
1.91 - break;
1.92 - }
1.93 - }
1.94 -}
1.95 -
1.96 -bool
1.97 -Scanner::eatComment()
1.98 -{
1.99 - char c;
1.100 -
1.101 - while ((c = getChar()) != '\0') {
1.102 - while (c == '*') {
1.103 - c = getChar();
1.104 - if (c == '\0')
1.105 - return false;
1.106 - if (c == '/')
1.107 - return true;
1.108 - }
1.109 - if (c == '\r' || c == '\n')
1.110 - calcNewLine(c);
1.111 - }
1.112 -
1.113 - return false;
1.114 -}
1.115 -
1.116 -static inline bool
1.117 -IsDigit(char c)
1.118 -{
1.119 - return c >= '0' && c <= '9';
1.120 -}
1.121 -
1.122 -// Based off the logic in sc2.c's ftoi()...
1.123 -double
1.124 -Scanner::ParseDouble(const char *string)
1.125 -{
1.126 - const char *ptr = string;
1.127 -
1.128 - double number = 0.0;
1.129 - while (IsDigit(*ptr)) {
1.130 - number = (number * 10) + (*ptr - '0');
1.131 - ptr++;
1.132 - }
1.133 -
1.134 - assert(*ptr == '.');
1.135 - ptr++;
1.136 -
1.137 - double fraction = 0.0;
1.138 - double multiplier = 1.0;
1.139 - while (IsDigit(*ptr)) {
1.140 - fraction = (fraction * 10) + (*ptr - '0');
1.141 - multiplier = multiplier / 10.0;
1.142 - ptr++;
1.143 - }
1.144 -
1.145 - number += fraction * multiplier;
1.146 -
1.147 - if (*ptr++ == 'e') {
1.148 - int sign = 1;
1.149 - if (*ptr == '-') {
1.150 - sign = -1;
1.151 - ptr++;
1.152 - }
1.153 -
1.154 - int exponent = 0;
1.155 - while (IsDigit(*ptr)) {
1.156 - exponent = (exponent * 10) + (*ptr - '0');
1.157 - ptr++;
1.158 - }
1.159 -
1.160 - multiplier = pow(10.0, exponent * sign);
1.161 - number *= multiplier;
1.162 - }
1.163 -
1.164 - return number;
1.165 -}
1.166 -
1.167 -static inline bool
1.168 -IsHexDigit(char c)
1.169 -{
1.170 - return IsDigit(c) ||
1.171 - (c >= 'a' && c <= 'f') ||
1.172 - (c >= 'A' && c <= 'F');
1.173 -}
1.174 -
1.175 -// Based on the logic for litchar() in sc2.c.
1.176 -int
1.177 -Scanner::readEscapeCode()
1.178 -{
1.179 - char c = getChar();
1.180 - if (c == '\\')
1.181 - return c;
1.182 -
1.183 - switch (c) {
1.184 - case 'a':
1.185 - return '\a';
1.186 - case 'b':
1.187 - return '\b';
1.188 - case 'e':
1.189 - return 27; // Apparently \e is non-standard.
1.190 - case 'f':
1.191 - return '\f';
1.192 - case 'n':
1.193 - return '\n';
1.194 - case 'r':
1.195 - return '\r';
1.196 - case 't':
1.197 - return '\t';
1.198 - case '\v':
1.199 - return '\v';
1.200 -
1.201 - case 'x':
1.202 - {
1.203 - unsigned digits = 0;
1.204 - char r = 0;
1.205 -
1.206 - c = getChar();
1.207 - while (IsHexDigit(c) && digits < 2) {
1.208 - if (IsDigit(c))
1.209 - c = (c << 4) + (c - '0');
1.210 - else
1.211 - c = (c << 4) + (tolower(c) - 'a' + 10);
1.212 - digits++;
1.213 - c = getChar();
1.214 - }
1.215 -
1.216 - // Swallow a trailing ';'
1.217 - if (c != ';')
1.218 - putChar(c);
1.219 -
1.220 - return r;
1.221 - }
1.222 -
1.223 - case '\'':
1.224 - case '\"':
1.225 - case '%':
1.226 - return c;
1.227 -
1.228 - default:
1.229 - {
1.230 - if (IsDigit(c)) {
1.231 - // \ddd
1.232 - char r = 0;
1.233 - while (IsDigit(c)) {
1.234 - r = r * 10 + (c - '0');
1.235 - c = getChar();
1.236 - }
1.237 -
1.238 - // Swallow a trailing ;
1.239 - if (c != ';')
1.240 - putChar(c);
1.241 -
1.242 - return r;
1.243 - }
1.244 - cc_.reportError(begin(), Message_UnknownEscapeCode, c);
1.245 - return INT_MAX;
1.246 - }
1.247 - }
1.248 -}
1.249 -
1.250 -TokenKind
1.251 -Scanner::stringLiteral()
1.252 -{
1.253 - literal_.clear();
1.254 -
1.255 - for (;;) {
1.256 - char c = getChar();
1.257 - if (c == '\"')
1.258 - break;
1.259 - if (c == '\r' || c == '\n' || c == '\0') {
1.260 - cc_.reportError(begin(), Message_UnterminatedString);
1.261 - return TOK_ERROR;
1.262 - }
1.263 - if (c == '\\') {
1.264 - int code = readEscapeCode();
1.265 - if (code == INT_MAX)
1.266 - return TOK_ERROR;
1.267 - c = char(code);
1.268 - }
1.269 - literal_ += c;
1.270 - }
1.271 -
1.272 - return TOK_STRING_LITERAL;
1.273 -}
1.274 -
1.275 -TokenKind
1.276 -Scanner::numberLiteral(char first)
1.277 -{
1.278 - literal_ = first;
1.279 -
1.280 - char c;
1.281 - for (;;) {
1.282 - c = getChar();
1.283 - if (!IsDigit(c))
1.284 - break;
1.285 - literal_ += c;
1.286 - }
1.287 -
1.288 - if (c != '.') {
1.289 - putChar(c);
1.290 - return TOK_INTEGER_LITERAL;
1.291 - }
1.292 - literal_ += c;
1.293 -
1.294 - c = getChar();
1.295 - if (!IsDigit(c)) {
1.296 - cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
1.297 - return TOK_ERROR;
1.298 - }
1.299 - literal_ += c;
1.300 -
1.301 - for (;;) {
1.302 - c = getChar();
1.303 - if (!IsDigit(c))
1.304 - break;
1.305 - literal_ += c;
1.306 - }
1.307 -
1.308 - if (c != 'e') {
1.309 - putChar(c);
1.310 - } else {
1.311 - literal_ += c;
1.312 - c = getChar();
1.313 - if (c == '-') {
1.314 - literal_ += c;
1.315 - c = getChar();
1.316 - }
1.317 - if (!IsDigit(c)) {
1.318 - cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
1.319 - return TOK_ERROR;
1.320 - }
1.321 - literal_ += c;
1.322 - for (;;) {
1.323 - c = getChar();
1.324 - if (!IsDigit(c)) {
1.325 - putChar(c);
1.326 - break;
1.327 - }
1.328 - literal_ += c;
1.329 - }
1.330 - }
1.331 -
1.332 - return TOK_FLOAT_LITERAL;
1.333 -}
1.334 -
1.335 -TokenKind
1.336 -Scanner::identifier(char first)
1.337 -{
1.338 - literal_ = first;
1.339 -
1.340 - char c;
1.341 - for (;;) {
1.342 - c = getChar();
1.343 - if (!isascii(c) || (!isalnum(c) && c != '_')) {
1.344 - if (c != ':')
1.345 - putChar(c);
1.346 - break;
1.347 - }
1.348 - literal_ += c;
1.349 - }
1.350 -
1.351 - if (literal_ == "new")
1.352 - return TOK_NEW;
1.353 - if (literal_ == "for")
1.354 - return TOK_FOR;
1.355 - if (literal_ == "return")
1.356 - return TOK_RETURN;
1.357 - if (literal_ == "if")
1.358 - return TOK_IF;
1.359 - if (literal_ == "else")
1.360 - return TOK_ELSE;
1.361 - if (literal_ == "enum")
1.362 - return TOK_ENUM;
1.363 - if (literal_ == "while")
1.364 - return TOK_WHILE;
1.365 - if (literal_ == "do")
1.366 - return TOK_DO;
1.367 - if (literal_ == "break")
1.368 - return TOK_BREAK;
1.369 - if (literal_ == "continue")
1.370 - return TOK_CONTINUE;
1.371 - if (literal_ == "forward")
1.372 - return TOK_FORWARD;
1.373 - if (literal_ == "public")
1.374 - return TOK_PUBLIC;
1.375 - if (literal_ == "const")
1.376 - return TOK_CONST;
1.377 - if (literal_ == "native")
1.378 - return TOK_NATIVE;
1.379 - return (c == ':') ? TOK_LABEL : TOK_NAME;
1.380 -}
1.381 -
1.382 -TokenKind
1.383 -Scanner::next()
1.384 -{
1.385 - assert(!pushed_);
1.386 -
1.387 - for (;;) {
1.388 - begin_ = cur_;
1.389 -
1.390 - char c = getChar();
1.391 - switch (c) {
1.392 - case '/':
1.393 - {
1.394 - c = getChar();
1.395 - if (c == '/') {
1.396 - eatSingleLine();
1.397 - break;
1.398 - }
1.399 - if (c == '*') {
1.400 - if (!eatComment()) {
1.401 - cc_.reportError(begin(), Message_UnterminatedComment);
1.402 - return TOK_ERROR;
1.403 - }
1.404 - break;
1.405 - }
1.406 - putChar(c);
1.407 - return TOK_SLASH;
1.408 - }
1.409 -
1.410 - case '\n':
1.411 - case '\r':
1.412 - calcNewLine(c);
1.413 - break;
1.414 -
1.415 - case ';': return TOK_SEMICOLON;
1.416 - case '{': return TOK_LBRACE;
1.417 - case '}': return TOK_RBRACE;
1.418 - case '(': return TOK_LPAREN;
1.419 - case ')': return TOK_RPAREN;
1.420 - case '[': return TOK_LBRACKET;
1.421 - case ']': return TOK_RBRACKET;
1.422 - case '~': return TOK_TILDE;
1.423 - case '?': return TOK_QMARK;
1.424 - case ':': return TOK_COLON;
1.425 -
1.426 - case '.':
1.427 - c = getChar();
1.428 - if (c == '.') {
1.429 - c = getChar();
1.430 - if (c == '.')
1.431 - return TOK_ELLIPSES;
1.432 - }
1.433 - cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
1.434 - return TOK_ERROR;
1.435 -
1.436 - case '*':
1.437 - c = getChar();
1.438 - if (c == '=')
1.439 - return TOK_ASSIGN_MUL;
1.440 - putChar(c);
1.441 - return TOK_STAR;
1.442 -
1.443 - case '+':
1.444 - c = getChar();
1.445 - if (c == '=')
1.446 - return TOK_ASSIGN_ADD;
1.447 - if (c == '+')
1.448 - return TOK_INCREMENT;
1.449 - putChar(c);
1.450 - return TOK_PLUS;
1.451 -
1.452 - case '&':
1.453 - c = getChar();
1.454 - if (c == '=')
1.455 - return TOK_ASSIGN_BITAND;
1.456 - if (c == '&')
1.457 - return TOK_AND;
1.458 - putChar(c);
1.459 - return TOK_BITAND;
1.460 -
1.461 - case '|':
1.462 - c = getChar();
1.463 - if (c == '=')
1.464 - return TOK_ASSIGN_BITOR;
1.465 - if (c == '|')
1.466 - return TOK_OR;
1.467 - putChar(c);
1.468 - return TOK_BITOR;
1.469 -
1.470 - case '^':
1.471 - c = getChar();
1.472 - if (c == '=')
1.473 - return TOK_ASSIGN_BITXOR;
1.474 - putChar(c);
1.475 - return TOK_BITXOR;
1.476 -
1.477 - case '%':
1.478 - c = getChar();
1.479 - if (c == '=')
1.480 - return TOK_ASSIGN_MOD;
1.481 - putChar(c);
1.482 - return TOK_PERCENT;
1.483 -
1.484 - case '-':
1.485 - c = getChar();
1.486 - if (c == '=')
1.487 - return TOK_ASSIGN_SUB;
1.488 - if (c == '-')
1.489 - return TOK_DECREMENT;
1.490 - putChar(c);
1.491 - return TOK_MINUS;
1.492 -
1.493 - case ',': return TOK_COMMA;
1.494 - case '\0': return TOK_EOF;
1.495 -
1.496 - case '!':
1.497 - c = getChar();
1.498 - if (c == '=')
1.499 - return TOK_NOTEQUALS;
1.500 - putChar(c);
1.501 - return TOK_NOT;
1.502 -
1.503 - case '=':
1.504 - c = getChar();
1.505 - if (c == '=')
1.506 - return TOK_EQUALS;
1.507 - putChar(c);
1.508 - return TOK_ASSIGN;
1.509 -
1.510 - case '<':
1.511 - c = getChar();
1.512 - if (c == '=')
1.513 - return TOK_LE;
1.514 - if (c == '<') {
1.515 - c = getChar();
1.516 - if (c == '=')
1.517 - return TOK_ASSIGN_SHL;
1.518 - return TOK_SHL;
1.519 - }
1.520 - putChar(c);
1.521 - return TOK_LT;
1.522 -
1.523 - case '>':
1.524 - c = getChar();
1.525 - if (c == '=')
1.526 - return TOK_GE;
1.527 - if (c == '>') {
1.528 - c = getChar();
1.529 - if (c == '>') {
1.530 - c = getChar();
1.531 - if (c == '=')
1.532 - return TOK_ASSIGN_USHR;
1.533 - putChar(c);
1.534 - return TOK_USHR;
1.535 - }
1.536 - return TOK_SHR;
1.537 - }
1.538 - putChar(c);
1.539 - return TOK_GT;
1.540 -
1.541 - case ' ':
1.542 - case '\t':
1.543 - case '\f':
1.544 - break;
1.545 -
1.546 - case '"':
1.547 - return stringLiteral();
1.548 -
1.549 - default:
1.550 - if (c >= '0' && c <= '9')
1.551 - return numberLiteral(c);
1.552 - if (c == '_' || (isascii(c) && isalpha(c)))
1.553 - return identifier(c);
1.554 - cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
1.555 - return TOK_ERROR;
1.556 - }
1.557 - }
1.558 -}
1.559 -
1.560 -TokenKind
1.561 -Scanner::scan()
1.562 -{
1.563 - if (pushed_) {
1.564 - pushed_ = false;
1.565 - return lastToken_;
1.566 - }
1.567 - lastToken_ = next();
1.568 - return lastToken_;
1.569 -}
1.570 -
1.571 -TokenKind
1.572 -Scanner::peek()
1.573 -{
1.574 - if (pushed_)
1.575 - return lastToken_;
1.576 - scan();
1.577 - pushed_ = true;
1.578 - return lastToken_;
1.579 -}
