diff -r 000000000000 -r c455ebb9a5dc jitcraft/Scanner.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jitcraft/Scanner.cpp	Mon Jun 25 02:57:28 2012 -0700
@@ -0,0 +1,227 @@
+/*
+ * Copyright (C) 2012 David Anderson
+ *
+ * This file is part of JITCraft.
+ *
+ * JITCraft is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option)
+ * any later version.
+ * 
+ * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * JITCraft. If not, see http://www.gnu.org/licenses/.
+ */
+#include <assert.h>
+#include <ctype.h>
+#include "Scanner.h"
+
+using namespace jc;
+
+const char *jc::TokenNames[] =
+{
+#define _(name, str) str,
+    TOKENMAP(_)
+#undef _
+    NULL
+};
+
+Scanner::Scanner(const char *stream, size_t length)
+  : stream_(stream),
+    length_(length),
+    pushed_(false),
+    saved_('\0'),
+    pos_(stream_)
+{
+}
+
+char
+Scanner::getChar()
+{
+    if (saved_ != '\0') {
+        char temp = saved_;
+        saved_ = '\0';
+        return temp;
+    }
+    if (pos_ >= stream_ + length_)
+        return '\0';
+    return *pos_++;
+}
+
+void
+Scanner::putChar(char c)
+{
+    assert(saved_ == '\0');
+    saved_ = c;
+}
+
+void
+Scanner::eatSingleLine()
+{
+    char c;
+    while ((c = getChar()) != '\0') {
+        if (c == '\n' || c == '\r')
+            break;
+    }
+}
+
+bool
+Scanner::eatComment()
+{
+    char c;
+
+    while ((c = getChar()) != '\0') {
+        while (c == '*') {
+            c = getChar();
+            if (c == '\0')
+                return false;
+            if (c == '/')
+                return true;
+        }
+    }
+
+    return false;
+}
+
+TokenKind
+Scanner::integerLiteral(char first)
+{
+    literal_ = first;
+
+    for (;;) {
+        char c = getChar();
+        if (c < '0' || c > '9') {
+            putChar(c);
+            break;
+        }
+        literal_ += c;
+    }
+
+    return TOK_INTEGER_LITERAL;
+}
+
+TokenKind
+Scanner::identifier(char first)
+{
+    literal_ = first;
+
+    for (;;) {
+        char c = getChar();
+        if (!isascii(c) || (!isalnum(c) && c != '_')) {
+            putChar(c);
+            break;
+        }
+        literal_ += c;
+    }
+
+    if (literal_ == "new")
+        return TOK_NEW;
+    if (literal_ == "for")
+        return TOK_FOR;
+    if (literal_ == "return")
+        return TOK_RETURN;
+    if (literal_ == "if")
+        return TOK_IF;
+    if (literal_ == "else")
+        return TOK_ELSE;
+    return TOK_NAME;
+}
+
+TokenKind
+Scanner::next()
+{
+    assert(!pushed_);
+
+    for (;;) {
+        char c = getChar();
+        switch (c) {
+          case '/':
+          {
+            c = getChar();
+            if (c == '/') {
+                eatSingleLine();
+                break;
+            }
+            if (c == '*') {
+                if (!eatComment())
+                    return TOK_ERROR;
+                break;
+            }
+            putChar(c);
+            return TOK_SLASH;
+          }
+
+          case ';': return TOK_SEMICOLON;
+          case '{': return TOK_LBRACE;
+          case '}': return TOK_RBRACE;
+          case '(': return TOK_LPAREN;
+          case ')': return TOK_RPAREN;
+          case '[': return TOK_LBRACKET;
+          case ']': return TOK_RBRACKET;
+          case '*': return TOK_STAR;
+          case '+': return TOK_PLUS;
+          case '-': return TOK_MINUS;
+          case ',': return TOK_COMMA;
+          case '\0': return TOK_EOF;
+
+          case '=':
+            c = getChar();
+            if (c == '=')
+                return TOK_EQUALS;
+            putChar(c);
+            return TOK_ASSIGN;
+
+          case '<':
+            c = getChar();
+            if (c == '=')
+                return TOK_LE;
+            putChar(c);
+            return TOK_LT;
+
+          case '>':
+            c = getChar();
+            if (c == '=')
+                return TOK_GE;
+            putChar(c);
+            return TOK_GT;
+
+          case ' ':
+          case '\t':
+          case '\r':
+          case '\f':
+          case '\n':
+            break;
+
+          default:
+            if (c >= '0' && c <= '9')
+                return integerLiteral(c);
+            if (c == '_' || (isascii(c) && isalpha(c)))
+                return identifier(c);
+            return TOK_ERROR;
+        }
+    }
+}
+
+TokenKind
+Scanner::scan()
+{
+    if (pushed_) {
+        pushed_ = false;
+        return lastToken_;
+    }
+    lastToken_ = next();
+    return lastToken_;
+}
+
+TokenKind
+Scanner::peek()
+{
+    if (pushed_)
+        return lastToken_;
+    scan();
+    pushed_ = true;
+    return lastToken_;
+}