diff -r 06a0015b6a5c -r 32ad0523b309 src/Scanner.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/Scanner.cpp	Tue Jul 24 23:53:34 2012 -0700
@@ -0,0 +1,576 @@
+/*
+ * Copyright (C) 2012 David Anderson
+ *
+ * This file is part of JITCraft.
+ *
+ * JITCraft is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * JITCraft. If not, see http://www.gnu.org/licenses/.
+ */
+#include <assert.h>
+#include <ctype.h>
+#include <math.h>
+#include "Scanner.h"
+#include "Utility.h"
+
+using namespace ke;
+
+const char *ke::TokenNames[] =
+{
+#define _(name, str) str,
+    TOKENMAP(_)
+#undef _
+    NULL
+};
+
+Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
+  : cc_(cc),
+    stream_(stream),
+    length_(length),
+    pushed_(false),
+    saved_('\0'),
+    pos_(stream_)
+{
+    cur_.line = 1;
+    cur_.col = 1;
+    last_ = cur_;
+}
+
+char
+Scanner::getChar()
+{
+    if (saved_ != '\0') {
+        char temp = saved_;
+        saved_ = '\0';
+        return temp;
+    }
+    if (pos_ >= stream_ + length_)
+        return '\0';
+    last_.col = cur_.col++;
+    return *pos_++;
+}
+
+void
+Scanner::putChar(char c)
+{
+    assert(saved_ == '\0');
+    saved_ = c;
+    cur_.col = last_.col;
+}
+
+void
+Scanner::calcNewLine(char c)
+{
+    cur_.line++;
+    cur_.col = 1;
+    last_ = cur_;
+
+    char o;
+    if (c == '\r' && ((o = getChar()) != '\n'))
+        putChar(o);
+}
+
+void
+Scanner::eatSingleLine()
+{
+    char c;
+    while ((c = getChar()) != '\0') {
+        if (c == '\n' || c == '\r') {
+            calcNewLine(c);
+            break;
+        }
+    }
+}
+
+bool
+Scanner::eatComment()
+{
+    char c;
+
+    while ((c = getChar()) != '\0') {
+        while (c == '*') {
+            c = getChar();
+            if (c == '\0')
+                return false;
+            if (c == '/')
+                return true;
+        }
+        if (c == '\r' || c == '\n')
+            calcNewLine(c);
+    }
+
+    return false;
+}
+
+static inline bool
+IsDigit(char c)
+{
+    return c >= '0' && c <= '9';
+}
+
+// Based off the logic in sc2.c's ftoi()...
+double
+Scanner::ParseDouble(const char *string)
+{
+    const char *ptr = string;
+
+    double number = 0.0;
+    while (IsDigit(*ptr)) {
+        number = (number * 10) + (*ptr - '0');
+        ptr++;
+    }
+
+    assert(*ptr == '.');
+    ptr++;
+
+    double fraction = 0.0;
+    double multiplier = 1.0;
+    while (IsDigit(*ptr)) {
+        fraction = (fraction * 10) + (*ptr - '0');
+        multiplier = multiplier / 10.0;
+        ptr++;
+    }
+
+    number += fraction * multiplier;
+
+    if (*ptr++ == 'e') {
+        int sign = 1;
+        if (*ptr == '-') {
+            sign = -1;
+            ptr++;
+        }
+
+        int exponent = 0;
+        while (IsDigit(*ptr)) {
+            exponent = (exponent * 10) + (*ptr - '0');
+            ptr++;
+        }
+
+        multiplier = pow(10.0, exponent * sign);
+        number *= multiplier;
+    }
+
+    return number;
+}
+
+static inline bool
+IsHexDigit(char c)
+{
+    return IsDigit(c) ||
+           (c >= 'a' && c <= 'f') ||
+           (c >= 'A' && c <= 'F');
+}
+
+// Based on the logic for litchar() in sc2.c.
+int
+Scanner::readEscapeCode()
+{
+    char c = getChar();
+    if (c == '\\')
+        return c;
+
+    switch (c) {
+      case 'a':
+        return '\a';
+      case 'b':
+        return '\b';
+      case 'e':
+        return 27;          // Apparently \e is non-standard.
+      case 'f':
+        return '\f';
+      case 'n':
+        return '\n';
+      case 'r':
+        return '\r';
+      case 't':
+        return '\t';
+      case '\v':
+        return '\v';
+
+      case 'x':
+      {
+          unsigned digits = 0;
+          char r = 0;
+  
+          c = getChar();
+          while (IsHexDigit(c) && digits < 2) {
+              if (IsDigit(c))
+                  c = (c << 4) + (c - '0');
+              else
+                  c = (c << 4) + (tolower(c) - 'a' + 10);
+              digits++;
+              c = getChar();
+           }
+                
+          // Swallow a trailing ';'
+          if (c != ';')
+              putChar(c);
+  
+          return r;
+      }
+
+      case '\'':
+      case '\"':
+      case '%':
+        return c;
+
+      default:
+      {
+        if (IsDigit(c)) {
+            // \ddd
+            char r = 0;
+            while (IsDigit(c)) {
+                r = r * 10 + (c - '0');
+                c = getChar();
+            }
+
+            // Swallow a trailing ;
+            if (c != ';')
+                putChar(c);
+
+            return r;
+        }
+        cc_.reportError(begin(), Message_UnknownEscapeCode, c);
+        return INT_MAX;
+      }
+    }
+}
+
+TokenKind
+Scanner::stringLiteral()
+{
+    literal_.clear();
+
+    for (;;) {
+        char c = getChar();
+        if (c == '\"')
+            break;
+        if (c == '\r' || c == '\n' || c == '\0') {
+            cc_.reportError(begin(), Message_UnterminatedString);
+            return TOK_ERROR;
+        }
+        if (c == '\\') {
+            int code = readEscapeCode();
+            if (code == INT_MAX)
+                return TOK_ERROR;
+            c = char(code);
+        }
+        literal_ += c;
+    }
+
+    return TOK_STRING_LITERAL;
+}
+
+TokenKind
+Scanner::numberLiteral(char first)
+{
+    literal_ = first;
+
+    char c;
+    for (;;) {
+        c = getChar();
+        if (!IsDigit(c))
+            break;
+        literal_ += c;
+    }
+
+    if (c != '.') {
+        putChar(c);
+        return TOK_INTEGER_LITERAL;
+    }
+    literal_ += c;
+
+    c = getChar();
+    if (!IsDigit(c)) {
+        cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
+        return TOK_ERROR;
+    }
+    literal_ += c;
+
+    for (;;) {
+        c = getChar();
+        if (!IsDigit(c))
+            break;
+        literal_ += c;
+    }
+
+    if (c != 'e') {
+        putChar(c);
+    } else {
+        literal_ += c;
+        c = getChar();
+        if (c == '-') {
+            literal_ += c;
+            c = getChar();
+        }
+        if (!IsDigit(c)) {
+            cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
+            return TOK_ERROR;
+        }
+        literal_ += c;
+        for (;;) {
+            c = getChar();
+            if (!IsDigit(c)) {
+                putChar(c);
+                break;
+            }
+            literal_ += c;
+        }
+    }
+
+    return TOK_FLOAT_LITERAL;
+}
+
+TokenKind
+Scanner::identifier(char first)
+{
+    literal_ = first;
+
+    char c;
+    for (;;) {
+        c = getChar();
+        if (!isascii(c) || (!isalnum(c) && c != '_')) {
+            if (c != ':')
+                putChar(c);
+            break;
+        }
+        literal_ += c;
+    }
+
+    if (literal_ == "new")
+        return TOK_NEW;
+    if (literal_ == "for")
+        return TOK_FOR;
+    if (literal_ == "return")
+        return TOK_RETURN;
+    if (literal_ == "if")
+        return TOK_IF;
+    if (literal_ == "else")
+        return TOK_ELSE;
+    if (literal_ == "enum")
+        return TOK_ENUM;
+    if (literal_ == "while")
+        return TOK_WHILE;
+    if (literal_ == "do")
+        return TOK_DO;
+    if (literal_ == "break")
+        return TOK_BREAK;
+    if (literal_ == "continue")
+        return TOK_CONTINUE;
+    if (literal_ == "forward")
+        return TOK_FORWARD;
+    if (literal_ == "public")
+        return TOK_PUBLIC;
+    if (literal_ == "const")
+        return TOK_CONST;
+    if (literal_ == "native")
+        return TOK_NATIVE;
+    return (c == ':') ? TOK_LABEL : TOK_NAME;
+}
+
+TokenKind
+Scanner::next()
+{
+    assert(!pushed_);
+
+    for (;;) {
+        begin_ = cur_;
+
+        char c = getChar();
+        switch (c) {
+          case '/':
+          {
+            c = getChar();
+            if (c == '/') {
+                eatSingleLine();
+                break;
+            }
+            if (c == '*') {
+                if (!eatComment()) {
+                    cc_.reportError(begin(), Message_UnterminatedComment);
+                    return TOK_ERROR;
+                }
+                break;
+            }
+            putChar(c);
+            return TOK_SLASH;
+          }
+
+          case '\n':
+          case '\r':
+            calcNewLine(c);
+            break;
+
+          case ';': return TOK_SEMICOLON;
+          case '{': return TOK_LBRACE;
+          case '}': return TOK_RBRACE;
+          case '(': return TOK_LPAREN;
+          case ')': return TOK_RPAREN;
+          case '[': return TOK_LBRACKET;
+          case ']': return TOK_RBRACKET;
+          case '~': return TOK_TILDE;
+          case '?': return TOK_QMARK;
+          case ':': return TOK_COLON;
+
+          case '.':
+            c = getChar();
+            if (c == '.') {
+                c = getChar();
+                if (c == '.')
+                    return TOK_ELLIPSES;
+            }
+            cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
+            return TOK_ERROR;
+
+          case '*':
+            c = getChar();
+            if (c == '=')
+                return TOK_ASSIGN_MUL;
+            putChar(c);
+            return TOK_STAR;
+
+          case '+':
+            c = getChar();
+            if (c == '=')
+                return TOK_ASSIGN_ADD;
+            if (c == '+')
+                return TOK_INCREMENT;
+            putChar(c);
+            return TOK_PLUS;
+
+          case '&':
+            c = getChar();
+            if (c == '=')
+                return TOK_ASSIGN_BITAND;
+            if (c == '&')
+                return TOK_AND;
+            putChar(c);
+            return TOK_BITAND;
+
+          case '|':
+            c = getChar();
+            if (c == '=')
+                return TOK_ASSIGN_BITOR;
+            if (c == '|')
+                return TOK_OR;
+            putChar(c);
+            return TOK_BITOR;
+
+          case '^':
+            c = getChar();
+            if (c == '=')
+                return TOK_ASSIGN_BITXOR;
+            putChar(c);
+            return TOK_BITXOR;
+
+          case '%':
+            c = getChar();
+            if (c == '=')
+                return TOK_ASSIGN_MOD;
+            putChar(c);
+            return TOK_PERCENT;
+
+          case '-':
+            c = getChar();
+            if (c == '=')
+                return TOK_ASSIGN_SUB;
+            if (c == '-')
+                return TOK_DECREMENT;
+            putChar(c);
+            return TOK_MINUS;
+
+          case ',': return TOK_COMMA;
+          case '\0': return TOK_EOF;
+
+          case '!':
+            c = getChar();
+            if (c == '=')
+                return TOK_NOTEQUALS;
+            putChar(c);
+            return TOK_NOT;
+
+          case '=':
+            c = getChar();
+            if (c == '=')
+                return TOK_EQUALS;
+            putChar(c);
+            return TOK_ASSIGN;
+
+          case '<':
+            c = getChar();
+            if (c == '=')
+                return TOK_LE;
+            if (c == '<') {
+                c = getChar();
+                if (c == '=')
+                    return TOK_ASSIGN_SHL;
+                return TOK_SHL;
+            }
+            putChar(c);
+            return TOK_LT;
+
+          case '>':
+            c = getChar();
+            if (c == '=')
+                return TOK_GE;
+            if (c == '>') {
+                c = getChar();
+                if (c == '>') {
+                    c = getChar();
+                    if (c == '=')
+                        return TOK_ASSIGN_USHR;
+                    putChar(c);
+                    return TOK_USHR;
+                }
+                return TOK_SHR;
+            }
+            putChar(c);
+            return TOK_GT;
+
+          case ' ':
+          case '\t':
+          case '\f':
+            break;
+
+          case '"':
+            return stringLiteral();
+
+          default:
+            if (c >= '0' && c <= '9')
+                return numberLiteral(c);
+            if (c == '_' || (isascii(c) && isalpha(c)))
+                return identifier(c);
+            cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
+            return TOK_ERROR;
+        }
+    }
+}
+
+TokenKind
+Scanner::scan()
+{
+    if (pushed_) {
+        pushed_ = false;
+        return lastToken_;
+    }
+    lastToken_ = next();
+    return lastToken_;
+}
+
+TokenKind
+Scanner::peek()
+{
+    if (pushed_)
+        return lastToken_;
+    scan();
+    pushed_ = true;
+    return lastToken_;
+}