diff -r b029d63565a2 -r c1ba166f3fc4 src/Scanner.cpp
--- a/src/Scanner.cpp	Mon Sep 17 00:10:01 2012 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,679 +0,0 @@
-/* vim: set ts=4 sw=4 tw=99 et:
- *
- * Copyright (C) 2012 David Anderson
- *
- * This file is part of JITCraft.
- *
- * JITCraft is free software: you can redistribute it and/or modify it under
- * the terms of the GNU General Public License as published by the Free
- * Software Foundation, either version 3 of the License, or (at your option)
- * any later version.
- * 
- * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * JITCraft. If not, see http://www.gnu.org/licenses/.
- */
-#include <assert.h>
-#include <ctype.h>
-#include <math.h>
-#include <limits.h>
-#include <string.h>
-#include "Scanner.h"
-#include "Utility.h"
-
-using namespace ke;
-
-const char *ke::TokenNames[] =
-{
-#define _(name, str) str,
-    TOKENMAP(_)
-#undef _
-    NULL
-};
-
-Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
-  : cc_(cc),
-    stream_(stream),
-    length_(length),
-    pushed_(false),
-    allowTags_(true),
-    saved_('\0'),
-    pos_(stream_)
-{
-    cur_.line = 1;
-    cur_.col = 1;
-    last_ = cur_;
-}
-
-char
-Scanner::getChar()
-{
-    if (saved_ != '\0') {
-        char temp = saved_;
-        saved_ = '\0';
-        return temp;
-    }
-    if (pos_ >= stream_ + length_)
-        return '\0';
-    last_.col = cur_.col++;
-    return *pos_++;
-}
-
-void
-Scanner::putChar(char c)
-{
-    assert(saved_ == '\0');
-    saved_ = c;
-    cur_.col = last_.col;
-}
-
-void
-Scanner::calcNewLine(char c)
-{
-    cur_.line++;
-    cur_.col = 1;
-    last_ = cur_;
-
-    char o;
-    if (c == '\r' && ((o = getChar()) != '\n'))
-        putChar(o);
-}
-
-void
-Scanner::eatSingleLine()
-{
-    char c;
-    while ((c = getChar()) != '\0') {
-        if (c == '\n' || c == '\r') {
-            calcNewLine(c);
-            break;
-        }
-    }
-}
-
-bool
-Scanner::eatComment()
-{
-    char c;
-
-    while ((c = getChar()) != '\0') {
-        while (c == '*') {
-            c = getChar();
-            if (c == '\0')
-                return false;
-            if (c == '/')
-                return true;
-        }
-        if (c == '\r' || c == '\n')
-            calcNewLine(c);
-    }
-
-    return false;
-}
-
-static inline bool
-IsDigit(char c)
-{
-    return c >= '0' && c <= '9';
-}
-
-// Based off the logic in sc2.c's ftoi()...
-double
-Scanner::ParseDouble(const char *string)
-{
-    const char *ptr = string;
-
-    double number = 0.0;
-    while (IsDigit(*ptr)) {
-        number = (number * 10) + (*ptr - '0');
-        ptr++;
-    }
-
-    assert(*ptr == '.');
-    ptr++;
-
-    double fraction = 0.0;
-    double multiplier = 1.0;
-    while (IsDigit(*ptr)) {
-        fraction = (fraction * 10) + (*ptr - '0');
-        multiplier = multiplier / 10.0;
-        ptr++;
-    }
-
-    number += fraction * multiplier;
-
-    if (*ptr++ == 'e') {
-        int sign = 1;
-        if (*ptr == '-') {
-            sign = -1;
-            ptr++;
-        }
-
-        int exponent = 0;
-        while (IsDigit(*ptr)) {
-            exponent = (exponent * 10) + (*ptr - '0');
-            ptr++;
-        }
-
-        multiplier = pow(10.0, exponent * sign);
-        number *= multiplier;
-    }
-
-    return number;
-}
-
-static inline bool
-IsHexDigit(char c)
-{
-    return IsDigit(c) ||
-           (c >= 'a' && c <= 'f') ||
-           (c >= 'A' && c <= 'F');
-}
-
-// Based on the logic for litchar() in sc2.c.
-int
-Scanner::readEscapeCode()
-{
-    char c = getChar();
-    if (c == '\\')
-        return c;
-
-    switch (c) {
-      case 'a':
-        return '\a';
-      case 'b':
-        return '\b';
-      case 'e':
-        return 27;          // Apparently \e is non-standard.
-      case 'f':
-        return '\f';
-      case 'n':
-        return '\n';
-      case 'r':
-        return '\r';
-      case 't':
-        return '\t';
-      case '\v':
-        return '\v';
-
-      case 'x':
-      {
-          unsigned digits = 0;
-          char r = 0;
-  
-          c = getChar();
-          while (IsHexDigit(c) && digits < 2) {
-              if (IsDigit(c))
-                  c = (c << 4) + (c - '0');
-              else
-                  c = (c << 4) + (tolower(c) - 'a' + 10);
-              digits++;
-              c = getChar();
-           }
-                
-          // Swallow a trailing ';'
-          if (c != ';')
-              putChar(c);
-  
-          return r;
-      }
-
-      case '\'':
-      case '\"':
-      case '%':
-        return c;
-
-      default:
-      {
-        if (IsDigit(c)) {
-            // \ddd
-            char r = 0;
-            while (IsDigit(c)) {
-                r = r * 10 + (c - '0');
-                c = getChar();
-            }
-
-            // Swallow a trailing ;
-            if (c != ';')
-                putChar(c);
-
-            return r;
-        }
-        cc_.reportError(begin(), Message_UnknownEscapeCode, c);
-        return INT_MAX;
-      }
-    }
-}
-
-TokenKind
-Scanner::stringLiteral()
-{
-    literal_.clear();
-
-    for (;;) {
-        char c = getChar();
-        if (c == '\"')
-            break;
-        if (c == '\r' || c == '\n' || c == '\0') {
-            cc_.reportError(begin(), Message_UnterminatedString);
-            return TOK_ERROR;
-        }
-        if (c == '\\') {
-            int code = readEscapeCode();
-            if (code == INT_MAX)
-                return TOK_ERROR;
-            c = char(code);
-        }
-        if (!literal_.append(c))
-            return TOK_ERROR;
-    }
-
-    if (!literal_.append('\0'))
-        return TOK_ERROR;
-
-    return TOK_STRING_LITERAL;
-}
-
-TokenKind
-Scanner::hexLiteral()
-{
-    literal_.clear();
-
-    char c;
-    for (;;) {
-        c = getChar();
-        if (!IsHexDigit(c))
-            break;
-        if (!literal_.append(c))
-            return TOK_ERROR;
-    }
-
-    putChar(c);
-
-    if (!literal_.append('\0'))
-        return TOK_ERROR;
-    return TOK_HEX_LITERAL;
-}
-
-TokenKind
-Scanner::numberLiteral(char first)
-{
-    literal_.clear();
-    if (!literal_.append(first))
-        return TOK_ERROR;
-
-    char c;
-    for (;;) {
-        c = getChar();
-        if (!IsDigit(c))
-            break;
-        if (!literal_.append(c))
-            return TOK_ERROR;
-    }
-
-    // Detect a hexadecimal string.
-    if (literal_.length() == 1 &&
-        literal_[0] == '0' &&
-        (c == 'x' || c == 'X'))
-    {
-        return hexLiteral();
-    }
-
-    if (c != '.') {
-        putChar(c);
-        if (!literal_.append('\0'))
-            return TOK_ERROR;
-        return TOK_INTEGER_LITERAL;
-    }
-    if (!literal_.append(c))
-        return TOK_ERROR;
-
-    c = getChar();
-    if (!IsDigit(c)) {
-        cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
-        return TOK_ERROR;
-    }
-    if (!literal_.append(c))
-        return TOK_ERROR;
-
-    for (;;) {
-        c = getChar();
-        if (!IsDigit(c))
-            break;
-        if (!literal_.append(c))
-            return TOK_ERROR;
-    }
-
-    if (c != 'e') {
-        putChar(c);
-    } else {
-        if (!literal_.append(c))
-            return TOK_ERROR;
-        c = getChar();
-        if (c == '-') {
-            if (!literal_.append(c))
-                return TOK_ERROR;
-            c = getChar();
-        }
-        if (!IsDigit(c)) {
-            cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
-            return TOK_ERROR;
-        }
-        if (!literal_.append(c))
-            return TOK_ERROR;
-        for (;;) {
-            c = getChar();
-            if (!IsDigit(c)) {
-                putChar(c);
-                break;
-            }
-            if (!literal_.append(c))
-                return TOK_ERROR;
-        }
-    }
-
-    if (!literal_.append('\0'))
-        return TOK_ERROR;
-
-    return TOK_FLOAT_LITERAL;
-}
-
-static inline TokenKind
-FindIdentifier(const char *str)
-{
-    if (!strcmp(str,  "new"))
-        return TOK_NEW;
-    if (!strcmp(str,  "for"))
-        return TOK_FOR;
-    if (!strcmp(str,  "return"))
-        return TOK_RETURN;
-    if (!strcmp(str,  "if"))
-        return TOK_IF;
-    if (!strcmp(str,  "else"))
-        return TOK_ELSE;
-    if (!strcmp(str,  "enum"))
-        return TOK_ENUM;
-    if (!strcmp(str,  "while"))
-        return TOK_WHILE;
-    if (!strcmp(str,  "do"))
-        return TOK_DO;
-    if (!strcmp(str,  "break"))
-        return TOK_BREAK;
-    if (!strcmp(str,  "continue"))
-        return TOK_CONTINUE;
-    if (!strcmp(str,  "forward"))
-        return TOK_FORWARD;
-    if (!strcmp(str,  "public"))
-        return TOK_PUBLIC;
-    if (!strcmp(str,  "const"))
-        return TOK_CONST;
-    if (!strcmp(str,  "native"))
-        return TOK_NATIVE;
-    if (!strcmp(str,  "stock"))
-        return TOK_STOCK;
-    if (!strcmp(str,  "true"))
-        return TOK_TRUE;
-    if (!strcmp(str,  "false"))
-        return TOK_FALSE;
-    if (!strcmp(str,  "struct"))
-        return TOK_STRUCT;
-    if (!strcmp(str,  "switch"))
-        return TOK_SWITCH;
-    if (!strcmp(str,  "case"))
-        return TOK_CASE;
-    if (!strcmp(str,  "default"))
-        return TOK_DEFAULT;
-    if (!strcmp(str,  "typedef"))
-        return TOK_TYPEDEF;
-    if (!strcmp(str,  "void"))
-        return TOK_VOID;
-    if (!strcmp(str,  "int"))
-        return TOK_INT;
-    if (!strcmp(str,  "sizeof"))
-        return TOK_SIZEOF;
-    return TOK_NONE;
-}
-
-TokenKind
-Scanner::identifier(char first)
-{
-    literal_.clear();
-    if (!literal_.append(first))
-        return TOK_ERROR;
-
-    char c;
-    for (;;) {
-        c = getChar();
-        if (!isascii(c) || (!isalnum(c) && c != '_'))
-            break;
-        if (!literal_.append(c))
-            return TOK_ERROR;
-    }
-    
-    if (!literal_.append('\0'))
-        return TOK_ERROR;
-
-    // Most keywords do not act as labels.
-    TokenKind kind = FindIdentifier(literal());
-    if (kind != TOK_NONE) {
-        putChar(c);
-        return kind;
-    }
-
-    if (c == ':' && allowTags_)
-        return TOK_LABEL;
-
-    putChar(c);
-
-    // _ is a special case which acts as a tag and a keyword.
-    if (!strcmp(literal(), "_"))
-        return TOK_UNDERSCORE;
-
-    return TOK_NAME;
-}
-
-TokenKind
-Scanner::next()
-{
-    assert(!pushed_);
-
-    for (;;) {
-        begin_ = cur_;
-
-        char c = getChar();
-        switch (c) {
-          case '/':
-          {
-            c = getChar();
-            if (c == '/') {
-                eatSingleLine();
-                break;
-            }
-            if (c == '*') {
-                if (!eatComment()) {
-                    cc_.reportError(begin(), Message_UnterminatedComment);
-                    return TOK_ERROR;
-                }
-                break;
-            }
-            putChar(c);
-            return TOK_SLASH;
-          }
-
-          case '\n':
-          case '\r':
-            calcNewLine(c);
-            break;
-
-          case ';': return TOK_SEMICOLON;
-          case '{': return TOK_LBRACE;
-          case '}': return TOK_RBRACE;
-          case '(': return TOK_LPAREN;
-          case ')': return TOK_RPAREN;
-          case '[': return TOK_LBRACKET;
-          case ']': return TOK_RBRACKET;
-          case '~': return TOK_TILDE;
-          case '?': return TOK_QMARK;
-          case ':': return TOK_COLON;
-
-          case '.':
-            c = getChar();
-            if (c == '.') {
-                c = getChar();
-                if (c == '.')
-                    return TOK_ELLIPSES;
-                cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
-                return TOK_ERROR;
-            }
-            putChar(c);
-            return TOK_DOT;
-
-          case '*':
-            c = getChar();
-            if (c == '=')
-                return TOK_ASSIGN_MUL;
-            putChar(c);
-            return TOK_STAR;
-
-          case '+':
-            c = getChar();
-            if (c == '=')
-                return TOK_ASSIGN_ADD;
-            if (c == '+')
-                return TOK_INCREMENT;
-            putChar(c);
-            return TOK_PLUS;
-
-          case '&':
-            c = getChar();
-            if (c == '=')
-                return TOK_ASSIGN_BITAND;
-            if (c == '&')
-                return TOK_AND;
-            putChar(c);
-            return TOK_BITAND;
-
-          case '|':
-            c = getChar();
-            if (c == '=')
-                return TOK_ASSIGN_BITOR;
-            if (c == '|')
-                return TOK_OR;
-            putChar(c);
-            return TOK_BITOR;
-
-          case '^':
-            c = getChar();
-            if (c == '=')
-                return TOK_ASSIGN_BITXOR;
-            putChar(c);
-            return TOK_BITXOR;
-
-          case '%':
-            c = getChar();
-            if (c == '=')
-                return TOK_ASSIGN_MOD;
-            putChar(c);
-            return TOK_PERCENT;
-
-          case '-':
-            c = getChar();
-            if (c == '=')
-                return TOK_ASSIGN_SUB;
-            if (c == '-')
-                return TOK_DECREMENT;
-            putChar(c);
-            return TOK_MINUS;
-
-          case ',': return TOK_COMMA;
-          case '\0': return TOK_EOF;
-
-          case '!':
-            c = getChar();
-            if (c == '=')
-                return TOK_NOTEQUALS;
-            putChar(c);
-            return TOK_NOT;
-
-          case '=':
-            c = getChar();
-            if (c == '=')
-                return TOK_EQUALS;
-            putChar(c);
-            return TOK_ASSIGN;
-
-
-          case '<':
-            c = getChar();
-            if (c == '=')
-                return TOK_LE;
-            if (c == '<') {
-                c = getChar();
-                if (c == '=')
-                    return TOK_ASSIGN_SHL;
-                putChar(c);
-                return TOK_SHL;
-            }
-            putChar(c);
-            return TOK_LT;
-
-          case '>':
-            c = getChar();
-            if (c == '=')
-                return TOK_GE;
-            if (c == '>') {
-                c = getChar();
-                if (c == '>') {
-                    c = getChar();
-                    if (c == '=')
-                        return TOK_ASSIGN_USHR;
-                    putChar(c);
-                    return TOK_USHR;
-                }
-                return TOK_SHR;
-            }
-            putChar(c);
-            return TOK_GT;
-
-          case ' ':
-          case '\t':
-          case '\f':
-            break;
-
-          case '"':
-            return stringLiteral();
-
-          default:
-            if (c >= '0' && c <= '9')
-                return numberLiteral(c);
-            if (c == '_' || (isascii(c) && isalpha(c)))
-                return identifier(c);
-            cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
-            return TOK_ERROR;
-        }
-    }
-}
-
-TokenKind
-Scanner::scan()
-{
-    if (pushed_) {
-        pushed_ = false;
-        return lastToken_;
-    }
-    lastToken_ = next();
-    return lastToken_;
-}
-
-TokenKind
-Scanner::peek()
-{
-    if (pushed_)
-        return lastToken_;
-    scan();
-    pushed_ = true;
-    return lastToken_;
-}