1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/src/Scanner.cpp Tue Jul 24 23:53:34 2012 -0700
1.3 @@ -0,0 +1,576 @@
1.4 +/*
1.5 + * Copyright (C) 2012 David Anderson
1.6 + *
1.7 + * This file is part of JITCraft.
1.8 + *
1.9 + * JITCraft is free software: you can redistribute it and/or modify it under
1.10 + * the terms of the GNU General Public License as published by the Free
1.11 + * Software Foundation, either version 3 of the License, or (at your option)
1.12 + * any later version.
1.13 + *
1.14 + * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
1.15 + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
1.16 + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
1.17 + *
1.18 + * You should have received a copy of the GNU General Public License along with
1.19 + * JITCraft. If not, see http://www.gnu.org/licenses/.
1.20 + */
1.21 +#include <assert.h>
1.22 +#include <ctype.h>
1.23 +#include <math.h>
1.24 +#include "Scanner.h"
1.25 +#include "Utility.h"
1.26 +
1.27 +using namespace ke;
1.28 +
1.29 +const char *ke::TokenNames[] =
1.30 +{
1.31 +#define _(name, str) str,
1.32 + TOKENMAP(_)
1.33 +#undef _
1.34 + NULL
1.35 +};
1.36 +
1.37 +Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
1.38 + : cc_(cc),
1.39 + stream_(stream),
1.40 + length_(length),
1.41 + pushed_(false),
1.42 + saved_('\0'),
1.43 + pos_(stream_)
1.44 +{
1.45 + cur_.line = 1;
1.46 + cur_.col = 1;
1.47 + last_ = cur_;
1.48 +}
1.49 +
1.50 +char
1.51 +Scanner::getChar()
1.52 +{
1.53 + if (saved_ != '\0') {
1.54 + char temp = saved_;
1.55 + saved_ = '\0';
1.56 + return temp;
1.57 + }
1.58 + if (pos_ >= stream_ + length_)
1.59 + return '\0';
1.60 + last_.col = cur_.col++;
1.61 + return *pos_++;
1.62 +}
1.63 +
1.64 +void
1.65 +Scanner::putChar(char c)
1.66 +{
1.67 + assert(saved_ == '\0');
1.68 + saved_ = c;
1.69 + cur_.col = last_.col;
1.70 +}
1.71 +
1.72 +void
1.73 +Scanner::calcNewLine(char c)
1.74 +{
1.75 + cur_.line++;
1.76 + cur_.col = 1;
1.77 + last_ = cur_;
1.78 +
1.79 + char o;
1.80 + if (c == '\r' && ((o = getChar()) != '\n'))
1.81 + putChar(o);
1.82 +}
1.83 +
1.84 +void
1.85 +Scanner::eatSingleLine()
1.86 +{
1.87 + char c;
1.88 + while ((c = getChar()) != '\0') {
1.89 + if (c == '\n' || c == '\r') {
1.90 + calcNewLine(c);
1.91 + break;
1.92 + }
1.93 + }
1.94 +}
1.95 +
1.96 +bool
1.97 +Scanner::eatComment()
1.98 +{
1.99 + char c;
1.100 +
1.101 + while ((c = getChar()) != '\0') {
1.102 + while (c == '*') {
1.103 + c = getChar();
1.104 + if (c == '\0')
1.105 + return false;
1.106 + if (c == '/')
1.107 + return true;
1.108 + }
1.109 + if (c == '\r' || c == '\n')
1.110 + calcNewLine(c);
1.111 + }
1.112 +
1.113 + return false;
1.114 +}
1.115 +
1.116 +static inline bool
1.117 +IsDigit(char c)
1.118 +{
1.119 + return c >= '0' && c <= '9';
1.120 +}
1.121 +
1.122 +// Based off the logic in sc2.c's ftoi()...
1.123 +double
1.124 +Scanner::ParseDouble(const char *string)
1.125 +{
1.126 + const char *ptr = string;
1.127 +
1.128 + double number = 0.0;
1.129 + while (IsDigit(*ptr)) {
1.130 + number = (number * 10) + (*ptr - '0');
1.131 + ptr++;
1.132 + }
1.133 +
1.134 + assert(*ptr == '.');
1.135 + ptr++;
1.136 +
1.137 + double fraction = 0.0;
1.138 + double multiplier = 1.0;
1.139 + while (IsDigit(*ptr)) {
1.140 + fraction = (fraction * 10) + (*ptr - '0');
1.141 + multiplier = multiplier / 10.0;
1.142 + ptr++;
1.143 + }
1.144 +
1.145 + number += fraction * multiplier;
1.146 +
1.147 + if (*ptr++ == 'e') {
1.148 + int sign = 1;
1.149 + if (*ptr == '-') {
1.150 + sign = -1;
1.151 + ptr++;
1.152 + }
1.153 +
1.154 + int exponent = 0;
1.155 + while (IsDigit(*ptr)) {
1.156 + exponent = (exponent * 10) + (*ptr - '0');
1.157 + ptr++;
1.158 + }
1.159 +
1.160 + multiplier = pow(10.0, exponent * sign);
1.161 + number *= multiplier;
1.162 + }
1.163 +
1.164 + return number;
1.165 +}
1.166 +
1.167 +static inline bool
1.168 +IsHexDigit(char c)
1.169 +{
1.170 + return IsDigit(c) ||
1.171 + (c >= 'a' && c <= 'f') ||
1.172 + (c >= 'A' && c <= 'F');
1.173 +}
1.174 +
1.175 +// Based on the logic for litchar() in sc2.c.
1.176 +int
1.177 +Scanner::readEscapeCode()
1.178 +{
1.179 + char c = getChar();
1.180 + if (c == '\\')
1.181 + return c;
1.182 +
1.183 + switch (c) {
1.184 + case 'a':
1.185 + return '\a';
1.186 + case 'b':
1.187 + return '\b';
1.188 + case 'e':
1.189 + return 27; // Apparently \e is non-standard.
1.190 + case 'f':
1.191 + return '\f';
1.192 + case 'n':
1.193 + return '\n';
1.194 + case 'r':
1.195 + return '\r';
1.196 + case 't':
1.197 + return '\t';
1.198 + case '\v':
1.199 + return '\v';
1.200 +
1.201 + case 'x':
1.202 + {
1.203 + unsigned digits = 0;
1.204 + char r = 0;
1.205 +
1.206 + c = getChar();
1.207 + while (IsHexDigit(c) && digits < 2) {
1.208 + if (IsDigit(c))
1.209 + c = (c << 4) + (c - '0');
1.210 + else
1.211 + c = (c << 4) + (tolower(c) - 'a' + 10);
1.212 + digits++;
1.213 + c = getChar();
1.214 + }
1.215 +
1.216 + // Swallow a trailing ';'
1.217 + if (c != ';')
1.218 + putChar(c);
1.219 +
1.220 + return r;
1.221 + }
1.222 +
1.223 + case '\'':
1.224 + case '\"':
1.225 + case '%':
1.226 + return c;
1.227 +
1.228 + default:
1.229 + {
1.230 + if (IsDigit(c)) {
1.231 + // \ddd
1.232 + char r = 0;
1.233 + while (IsDigit(c)) {
1.234 + r = r * 10 + (c - '0');
1.235 + c = getChar();
1.236 + }
1.237 +
1.238 + // Swallow a trailing ;
1.239 + if (c != ';')
1.240 + putChar(c);
1.241 +
1.242 + return r;
1.243 + }
1.244 + cc_.reportError(begin(), Message_UnknownEscapeCode, c);
1.245 + return INT_MAX;
1.246 + }
1.247 + }
1.248 +}
1.249 +
1.250 +TokenKind
1.251 +Scanner::stringLiteral()
1.252 +{
1.253 + literal_.clear();
1.254 +
1.255 + for (;;) {
1.256 + char c = getChar();
1.257 + if (c == '\"')
1.258 + break;
1.259 + if (c == '\r' || c == '\n' || c == '\0') {
1.260 + cc_.reportError(begin(), Message_UnterminatedString);
1.261 + return TOK_ERROR;
1.262 + }
1.263 + if (c == '\\') {
1.264 + int code = readEscapeCode();
1.265 + if (code == INT_MAX)
1.266 + return TOK_ERROR;
1.267 + c = char(code);
1.268 + }
1.269 + literal_ += c;
1.270 + }
1.271 +
1.272 + return TOK_STRING_LITERAL;
1.273 +}
1.274 +
1.275 +TokenKind
1.276 +Scanner::numberLiteral(char first)
1.277 +{
1.278 + literal_ = first;
1.279 +
1.280 + char c;
1.281 + for (;;) {
1.282 + c = getChar();
1.283 + if (!IsDigit(c))
1.284 + break;
1.285 + literal_ += c;
1.286 + }
1.287 +
1.288 + if (c != '.') {
1.289 + putChar(c);
1.290 + return TOK_INTEGER_LITERAL;
1.291 + }
1.292 + literal_ += c;
1.293 +
1.294 + c = getChar();
1.295 + if (!IsDigit(c)) {
1.296 + cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
1.297 + return TOK_ERROR;
1.298 + }
1.299 + literal_ += c;
1.300 +
1.301 + for (;;) {
1.302 + c = getChar();
1.303 + if (!IsDigit(c))
1.304 + break;
1.305 + literal_ += c;
1.306 + }
1.307 +
1.308 + if (c != 'e') {
1.309 + putChar(c);
1.310 + } else {
1.311 + literal_ += c;
1.312 + c = getChar();
1.313 + if (c == '-') {
1.314 + literal_ += c;
1.315 + c = getChar();
1.316 + }
1.317 + if (!IsDigit(c)) {
1.318 + cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
1.319 + return TOK_ERROR;
1.320 + }
1.321 + literal_ += c;
1.322 + for (;;) {
1.323 + c = getChar();
1.324 + if (!IsDigit(c)) {
1.325 + putChar(c);
1.326 + break;
1.327 + }
1.328 + literal_ += c;
1.329 + }
1.330 + }
1.331 +
1.332 + return TOK_FLOAT_LITERAL;
1.333 +}
1.334 +
1.335 +TokenKind
1.336 +Scanner::identifier(char first)
1.337 +{
1.338 + literal_ = first;
1.339 +
1.340 + char c;
1.341 + for (;;) {
1.342 + c = getChar();
1.343 + if (!isascii(c) || (!isalnum(c) && c != '_')) {
1.344 + if (c != ':')
1.345 + putChar(c);
1.346 + break;
1.347 + }
1.348 + literal_ += c;
1.349 + }
1.350 +
1.351 + if (literal_ == "new")
1.352 + return TOK_NEW;
1.353 + if (literal_ == "for")
1.354 + return TOK_FOR;
1.355 + if (literal_ == "return")
1.356 + return TOK_RETURN;
1.357 + if (literal_ == "if")
1.358 + return TOK_IF;
1.359 + if (literal_ == "else")
1.360 + return TOK_ELSE;
1.361 + if (literal_ == "enum")
1.362 + return TOK_ENUM;
1.363 + if (literal_ == "while")
1.364 + return TOK_WHILE;
1.365 + if (literal_ == "do")
1.366 + return TOK_DO;
1.367 + if (literal_ == "break")
1.368 + return TOK_BREAK;
1.369 + if (literal_ == "continue")
1.370 + return TOK_CONTINUE;
1.371 + if (literal_ == "forward")
1.372 + return TOK_FORWARD;
1.373 + if (literal_ == "public")
1.374 + return TOK_PUBLIC;
1.375 + if (literal_ == "const")
1.376 + return TOK_CONST;
1.377 + if (literal_ == "native")
1.378 + return TOK_NATIVE;
1.379 + return (c == ':') ? TOK_LABEL : TOK_NAME;
1.380 +}
1.381 +
1.382 +TokenKind
1.383 +Scanner::next()
1.384 +{
1.385 + assert(!pushed_);
1.386 +
1.387 + for (;;) {
1.388 + begin_ = cur_;
1.389 +
1.390 + char c = getChar();
1.391 + switch (c) {
1.392 + case '/':
1.393 + {
1.394 + c = getChar();
1.395 + if (c == '/') {
1.396 + eatSingleLine();
1.397 + break;
1.398 + }
1.399 + if (c == '*') {
1.400 + if (!eatComment()) {
1.401 + cc_.reportError(begin(), Message_UnterminatedComment);
1.402 + return TOK_ERROR;
1.403 + }
1.404 + break;
1.405 + }
1.406 + putChar(c);
1.407 + return TOK_SLASH;
1.408 + }
1.409 +
1.410 + case '\n':
1.411 + case '\r':
1.412 + calcNewLine(c);
1.413 + break;
1.414 +
1.415 + case ';': return TOK_SEMICOLON;
1.416 + case '{': return TOK_LBRACE;
1.417 + case '}': return TOK_RBRACE;
1.418 + case '(': return TOK_LPAREN;
1.419 + case ')': return TOK_RPAREN;
1.420 + case '[': return TOK_LBRACKET;
1.421 + case ']': return TOK_RBRACKET;
1.422 + case '~': return TOK_TILDE;
1.423 + case '?': return TOK_QMARK;
1.424 + case ':': return TOK_COLON;
1.425 +
1.426 + case '.':
1.427 + c = getChar();
1.428 + if (c == '.') {
1.429 + c = getChar();
1.430 + if (c == '.')
1.431 + return TOK_ELLIPSES;
1.432 + }
1.433 + cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
1.434 + return TOK_ERROR;
1.435 +
1.436 + case '*':
1.437 + c = getChar();
1.438 + if (c == '=')
1.439 + return TOK_ASSIGN_MUL;
1.440 + putChar(c);
1.441 + return TOK_STAR;
1.442 +
1.443 + case '+':
1.444 + c = getChar();
1.445 + if (c == '=')
1.446 + return TOK_ASSIGN_ADD;
1.447 + if (c == '+')
1.448 + return TOK_INCREMENT;
1.449 + putChar(c);
1.450 + return TOK_PLUS;
1.451 +
1.452 + case '&':
1.453 + c = getChar();
1.454 + if (c == '=')
1.455 + return TOK_ASSIGN_BITAND;
1.456 + if (c == '&')
1.457 + return TOK_AND;
1.458 + putChar(c);
1.459 + return TOK_BITAND;
1.460 +
1.461 + case '|':
1.462 + c = getChar();
1.463 + if (c == '=')
1.464 + return TOK_ASSIGN_BITOR;
1.465 + if (c == '|')
1.466 + return TOK_OR;
1.467 + putChar(c);
1.468 + return TOK_BITOR;
1.469 +
1.470 + case '^':
1.471 + c = getChar();
1.472 + if (c == '=')
1.473 + return TOK_ASSIGN_BITXOR;
1.474 + putChar(c);
1.475 + return TOK_BITXOR;
1.476 +
1.477 + case '%':
1.478 + c = getChar();
1.479 + if (c == '=')
1.480 + return TOK_ASSIGN_MOD;
1.481 + putChar(c);
1.482 + return TOK_PERCENT;
1.483 +
1.484 + case '-':
1.485 + c = getChar();
1.486 + if (c == '=')
1.487 + return TOK_ASSIGN_SUB;
1.488 + if (c == '-')
1.489 + return TOK_DECREMENT;
1.490 + putChar(c);
1.491 + return TOK_MINUS;
1.492 +
1.493 + case ',': return TOK_COMMA;
1.494 + case '\0': return TOK_EOF;
1.495 +
1.496 + case '!':
1.497 + c = getChar();
1.498 + if (c == '=')
1.499 + return TOK_NOTEQUALS;
1.500 + putChar(c);
1.501 + return TOK_NOT;
1.502 +
1.503 + case '=':
1.504 + c = getChar();
1.505 + if (c == '=')
1.506 + return TOK_EQUALS;
1.507 + putChar(c);
1.508 + return TOK_ASSIGN;
1.509 +
1.510 + case '<':
1.511 + c = getChar();
1.512 + if (c == '=')
1.513 + return TOK_LE;
1.514 + if (c == '<') {
1.515 + c = getChar();
1.516 + if (c == '=')
1.517 + return TOK_ASSIGN_SHL;
1.518 + return TOK_SHL;
1.519 + }
1.520 + putChar(c);
1.521 + return TOK_LT;
1.522 +
1.523 + case '>':
1.524 + c = getChar();
1.525 + if (c == '=')
1.526 + return TOK_GE;
1.527 + if (c == '>') {
1.528 + c = getChar();
1.529 + if (c == '>') {
1.530 + c = getChar();
1.531 + if (c == '=')
1.532 + return TOK_ASSIGN_USHR;
1.533 + putChar(c);
1.534 + return TOK_USHR;
1.535 + }
1.536 + return TOK_SHR;
1.537 + }
1.538 + putChar(c);
1.539 + return TOK_GT;
1.540 +
1.541 + case ' ':
1.542 + case '\t':
1.543 + case '\f':
1.544 + break;
1.545 +
1.546 + case '"':
1.547 + return stringLiteral();
1.548 +
1.549 + default:
1.550 + if (c >= '0' && c <= '9')
1.551 + return numberLiteral(c);
1.552 + if (c == '_' || (isascii(c) && isalpha(c)))
1.553 + return identifier(c);
1.554 + cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
1.555 + return TOK_ERROR;
1.556 + }
1.557 + }
1.558 +}
1.559 +
1.560 +TokenKind
1.561 +Scanner::scan()
1.562 +{
1.563 + if (pushed_) {
1.564 + pushed_ = false;
1.565 + return lastToken_;
1.566 + }
1.567 + lastToken_ = next();
1.568 + return lastToken_;
1.569 +}
1.570 +
1.571 +TokenKind
1.572 +Scanner::peek()
1.573 +{
1.574 + if (pushed_)
1.575 + return lastToken_;
1.576 + scan();
1.577 + pushed_ = true;
1.578 + return lastToken_;
1.579 +}
