danderson@92: /* vim: set ts=4 sw=4 tw=99 et:
danderson@92:  *
dvander@0:  * Copyright (C) 2012 David Anderson
dvander@0:  *
dvander@0:  * This file is part of JITCraft.
dvander@0:  *
dvander@0:  * JITCraft is free software: you can redistribute it and/or modify it under
dvander@0:  * the terms of the GNU General Public License as published by the Free
dvander@0:  * Software Foundation, either version 3 of the License, or (at your option)
dvander@0:  * any later version.
dvander@0:  * 
dvander@0:  * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
dvander@0:  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
dvander@0:  * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
dvander@0:  *
dvander@0:  * You should have received a copy of the GNU General Public License along with
dvander@0:  * JITCraft. If not, see http://www.gnu.org/licenses/.
dvander@0:  */
dvander@0: #include <assert.h>
dvander@0: #include <ctype.h>
dvander@28: #include <math.h>
danderson@92: #include <limits.h>
danderson@92: #include <string.h>
dvander@195: #include "../Utility.h"
dvander@196: #include "../Heap-inl.h"
dvander@254: #include "../Handles-inl.h"
dvander@0: #include "Scanner.h"
dvander@0: 
dvander@59: using namespace ke;
dvander@0: 
dvander@59: const char *ke::TokenNames[] =
dvander@0: {
dvander@0: #define _(name, str) str,
dvander@0:     TOKENMAP(_)
dvander@0: #undef _
dvander@0:     NULL
dvander@0: };
dvander@0: 
dvander@25: Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
dvander@25:   : cc_(cc),
dvander@25:     stream_(stream),
dvander@0:     length_(length),
dvander@0:     pushed_(false),
dvander@172:     allowTags_(true),
dvander@0:     saved_('\0'),
dvander@0:     pos_(stream_)
dvander@0: {
dvander@47:     cur_.line = 1;
dvander@71:     cur_.col = 1;
dvander@25:     last_ = cur_;
dvander@0: }
dvander@0: 
dvander@0: char
dvander@0: Scanner::getChar()
dvander@0: {
dvander@0:     if (saved_ != '\0') {
dvander@0:         char temp = saved_;
dvander@0:         saved_ = '\0';
dvander@0:         return temp;
dvander@0:     }
dvander@0:     if (pos_ >= stream_ + length_)
dvander@0:         return '\0';
dvander@25:     last_.col = cur_.col++;
dvander@0:     return *pos_++;
dvander@0: }
dvander@0: 
dvander@0: void
dvander@0: Scanner::putChar(char c)
dvander@0: {
dvander@0:     assert(saved_ == '\0');
dvander@0:     saved_ = c;
dvander@25:     cur_.col = last_.col;
dvander@25: }
dvander@25: 
dvander@25: void
dvander@25: Scanner::calcNewLine(char c)
dvander@25: {
dvander@25:     cur_.line++;
dvander@71:     cur_.col = 1;
dvander@25:     last_ = cur_;
dvander@25: 
dvander@25:     char o;
dvander@25:     if (c == '\r' && ((o = getChar()) != '\n'))
dvander@25:         putChar(o);
dvander@0: }
dvander@0: 
dvander@0: void
dvander@0: Scanner::eatSingleLine()
dvander@0: {
dvander@0:     char c;
dvander@0:     while ((c = getChar()) != '\0') {
dvander@25:         if (c == '\n' || c == '\r') {
dvander@25:             calcNewLine(c);
dvander@0:             break;
dvander@25:         }
dvander@0:     }
dvander@0: }
dvander@0: 
dvander@0: bool
dvander@0: Scanner::eatComment()
dvander@0: {
dvander@0:     char c;
dvander@0: 
dvander@0:     while ((c = getChar()) != '\0') {
dvander@0:         while (c == '*') {
dvander@0:             c = getChar();
dvander@0:             if (c == '\0')
dvander@0:                 return false;
dvander@0:             if (c == '/')
dvander@0:                 return true;
dvander@0:         }
dvander@25:         if (c == '\r' || c == '\n')
dvander@25:             calcNewLine(c);
dvander@0:     }
dvander@0: 
dvander@0:     return false;
dvander@0: }
dvander@0: 
dvander@4: static inline bool
dvander@4: IsDigit(char c)
dvander@4: {
dvander@4:     return c >= '0' && c <= '9';
dvander@4: }
dvander@4: 
dvander@4: // Based off the logic in sc2.c's ftoi()...
dvander@4: double
dvander@4: Scanner::ParseDouble(const char *string)
dvander@4: {
dvander@4:     const char *ptr = string;
dvander@4: 
dvander@4:     double number = 0.0;
dvander@4:     while (IsDigit(*ptr)) {
dvander@4:         number = (number * 10) + (*ptr - '0');
dvander@4:         ptr++;
dvander@4:     }
dvander@4: 
dvander@4:     assert(*ptr == '.');
dvander@4:     ptr++;
dvander@4: 
dvander@4:     double fraction = 0.0;
dvander@4:     double multiplier = 1.0;
dvander@4:     while (IsDigit(*ptr)) {
dvander@4:         fraction = (fraction * 10) + (*ptr - '0');
dvander@4:         multiplier = multiplier / 10.0;
dvander@4:         ptr++;
dvander@4:     }
dvander@4: 
dvander@4:     number += fraction * multiplier;
dvander@4: 
dvander@4:     if (*ptr++ == 'e') {
dvander@4:         int sign = 1;
dvander@4:         if (*ptr == '-') {
dvander@4:             sign = -1;
dvander@4:             ptr++;
dvander@4:         }
dvander@4: 
dvander@4:         int exponent = 0;
dvander@4:         while (IsDigit(*ptr)) {
dvander@4:             exponent = (exponent * 10) + (*ptr - '0');
dvander@4:             ptr++;
dvander@4:         }
dvander@4: 
dvander@4:         multiplier = pow(10.0, exponent * sign);
dvander@4:         number *= multiplier;
dvander@4:     }
dvander@4: 
dvander@4:     return number;
dvander@4: }
dvander@4: 
dvander@21: static inline bool
dvander@21: IsHexDigit(char c)
dvander@21: {
dvander@21:     return IsDigit(c) ||
dvander@21:            (c >= 'a' && c <= 'f') ||
dvander@21:            (c >= 'A' && c <= 'F');
dvander@21: }
dvander@21: 
dvander@21: // Based on the logic for litchar() in sc2.c.
dvander@21: int
dvander@21: Scanner::readEscapeCode()
dvander@21: {
dvander@21:     char c = getChar();
dvander@21:     if (c == '\\')
dvander@21:         return c;
dvander@21: 
dvander@21:     switch (c) {
dvander@21:       case 'a':
dvander@21:         return '\a';
dvander@21:       case 'b':
dvander@21:         return '\b';
dvander@21:       case 'e':
dvander@21:         return 27;          // Apparently \e is non-standard.
dvander@21:       case 'f':
dvander@21:         return '\f';
dvander@21:       case 'n':
dvander@21:         return '\n';
dvander@21:       case 'r':
dvander@21:         return '\r';
dvander@21:       case 't':
dvander@21:         return '\t';
dvander@251:       case 'v':
dvander@21:         return '\v';
dvander@21: 
dvander@21:       case 'x':
dvander@21:       {
dvander@21:           unsigned digits = 0;
dvander@21:           char r = 0;
dvander@21:   
dvander@21:           c = getChar();
dvander@21:           while (IsHexDigit(c) && digits < 2) {
dvander@21:               if (IsDigit(c))
dvander@21:                   c = (c << 4) + (c - '0');
dvander@21:               else
dvander@21:                   c = (c << 4) + (tolower(c) - 'a' + 10);
dvander@21:               digits++;
dvander@21:               c = getChar();
dvander@21:            }
dvander@21:                 
dvander@21:           // Swallow a trailing ';'
dvander@21:           if (c != ';')
dvander@21:               putChar(c);
dvander@21:   
dvander@21:           return r;
dvander@21:       }
dvander@21: 
dvander@21:       case '\'':
dvander@21:       case '\"':
dvander@21:       case '%':
dvander@21:         return c;
dvander@21: 
dvander@21:       default:
dvander@21:       {
dvander@21:         if (IsDigit(c)) {
dvander@21:             // \ddd
dvander@21:             char r = 0;
dvander@21:             while (IsDigit(c)) {
dvander@21:                 r = r * 10 + (c - '0');
dvander@21:                 c = getChar();
dvander@21:             }
dvander@21: 
dvander@21:             // Swallow a trailing ;
dvander@21:             if (c != ';')
dvander@21:                 putChar(c);
dvander@21: 
dvander@21:             return r;
dvander@21:         }
dvander@25:         cc_.reportError(begin(), Message_UnknownEscapeCode, c);
dvander@21:         return INT_MAX;
dvander@21:       }
dvander@21:     }
dvander@21: }
dvander@21: 
dvander@21: TokenKind
dvander@21: Scanner::stringLiteral()
dvander@21: {
dvander@21:     literal_.clear();
dvander@21: 
dvander@21:     for (;;) {
dvander@21:         char c = getChar();
dvander@21:         if (c == '\"')
dvander@21:             break;
dvander@21:         if (c == '\r' || c == '\n' || c == '\0') {
dvander@25:             cc_.reportError(begin(), Message_UnterminatedString);
dvander@21:             return TOK_ERROR;
dvander@21:         }
dvander@21:         if (c == '\\') {
dvander@21:             int code = readEscapeCode();
dvander@21:             if (code == INT_MAX)
dvander@21:                 return TOK_ERROR;
dvander@21:             c = char(code);
dvander@21:         }
danderson@92:         if (!literal_.append(c))
danderson@92:             return TOK_ERROR;
dvander@21:     }
dvander@21: 
danderson@92:     if (!literal_.append('\0'))
danderson@92:         return TOK_ERROR;
danderson@92: 
dvander@21:     return TOK_STRING_LITERAL;
dvander@21: }
dvander@21: 
dvander@0: TokenKind
dvander@176: Scanner::hexLiteral()
dvander@176: {
dvander@176:     literal_.clear();
dvander@176: 
dvander@176:     char c;
dvander@176:     for (;;) {
dvander@176:         c = getChar();
dvander@176:         if (!IsHexDigit(c))
dvander@176:             break;
dvander@176:         if (!literal_.append(c))
dvander@176:             return TOK_ERROR;
dvander@176:     }
dvander@176: 
dvander@176:     putChar(c);
dvander@176: 
dvander@176:     if (!literal_.append('\0'))
dvander@176:         return TOK_ERROR;
dvander@176:     return TOK_HEX_LITERAL;
dvander@176: }
dvander@176: 
dvander@176: TokenKind
dvander@4: Scanner::numberLiteral(char first)
dvander@0: {
danderson@92:     literal_.clear();
danderson@92:     if (!literal_.append(first))
danderson@92:         return TOK_ERROR;
dvander@0: 
dvander@4:     char c;
dvander@0:     for (;;) {
dvander@4:         c = getChar();
dvander@4:         if (!IsDigit(c))
dvander@0:             break;
danderson@92:         if (!literal_.append(c))
danderson@92:             return TOK_ERROR;
dvander@0:     }
dvander@0: 
dvander@176:     // Detect a hexadecimal string.
dvander@176:     if (literal_.length() == 1 &&
dvander@176:         literal_[0] == '0' &&
dvander@176:         (c == 'x' || c == 'X'))
dvander@176:     {
dvander@176:         return hexLiteral();
dvander@176:     }
dvander@176: 
dvander@4:     if (c != '.') {
dvander@4:         putChar(c);
danderson@92:         if (!literal_.append('\0'))
danderson@92:             return TOK_ERROR;
dvander@4:         return TOK_INTEGER_LITERAL;
dvander@4:     }
danderson@92:     if (!literal_.append(c))
danderson@92:         return TOK_ERROR;
dvander@4: 
dvander@4:     c = getChar();
dvander@4:     if (!IsDigit(c)) {
dvander@25:         cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
dvander@4:         return TOK_ERROR;
dvander@4:     }
danderson@92:     if (!literal_.append(c))
danderson@92:         return TOK_ERROR;
dvander@4: 
dvander@4:     for (;;) {
dvander@4:         c = getChar();
dvander@4:         if (!IsDigit(c))
dvander@4:             break;
danderson@92:         if (!literal_.append(c))
danderson@92:             return TOK_ERROR;
dvander@4:     }
dvander@4: 
dvander@4:     if (c != 'e') {
dvander@4:         putChar(c);
dvander@4:     } else {
danderson@92:         if (!literal_.append(c))
danderson@92:             return TOK_ERROR;
dvander@4:         c = getChar();
dvander@4:         if (c == '-') {
danderson@92:             if (!literal_.append(c))
danderson@92:                 return TOK_ERROR;
dvander@4:             c = getChar();
dvander@4:         }
dvander@4:         if (!IsDigit(c)) {
dvander@25:             cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
dvander@4:             return TOK_ERROR;
dvander@4:         }
danderson@92:         if (!literal_.append(c))
danderson@92:             return TOK_ERROR;
dvander@4:         for (;;) {
dvander@4:             c = getChar();
dvander@4:             if (!IsDigit(c)) {
dvander@4:                 putChar(c);
dvander@4:                 break;
dvander@4:             }
danderson@92:             if (!literal_.append(c))
danderson@92:                 return TOK_ERROR;
dvander@4:         }
dvander@4:     }
dvander@4: 
danderson@92:     if (!literal_.append('\0'))
danderson@92:         return TOK_ERROR;
danderson@92: 
dvander@4:     return TOK_FLOAT_LITERAL;
dvander@0: }
dvander@0: 
dvander@171: static inline TokenKind
dvander@171: FindIdentifier(const char *str)
dvander@171: {
dvander@171:     if (!strcmp(str,  "new"))
dvander@171:         return TOK_NEW;
dvander@171:     if (!strcmp(str,  "for"))
dvander@171:         return TOK_FOR;
dvander@171:     if (!strcmp(str,  "return"))
dvander@171:         return TOK_RETURN;
dvander@171:     if (!strcmp(str,  "if"))
dvander@171:         return TOK_IF;
dvander@171:     if (!strcmp(str,  "else"))
dvander@171:         return TOK_ELSE;
dvander@171:     if (!strcmp(str,  "enum"))
dvander@171:         return TOK_ENUM;
dvander@171:     if (!strcmp(str,  "while"))
dvander@171:         return TOK_WHILE;
dvander@171:     if (!strcmp(str,  "do"))
dvander@171:         return TOK_DO;
dvander@171:     if (!strcmp(str,  "break"))
dvander@171:         return TOK_BREAK;
dvander@171:     if (!strcmp(str,  "continue"))
dvander@171:         return TOK_CONTINUE;
dvander@171:     if (!strcmp(str,  "forward"))
dvander@171:         return TOK_FORWARD;
dvander@171:     if (!strcmp(str,  "public"))
dvander@171:         return TOK_PUBLIC;
dvander@171:     if (!strcmp(str,  "const"))
dvander@171:         return TOK_CONST;
dvander@171:     if (!strcmp(str,  "native"))
dvander@171:         return TOK_NATIVE;
dvander@171:     if (!strcmp(str,  "stock"))
dvander@171:         return TOK_STOCK;
dvander@171:     if (!strcmp(str,  "true"))
dvander@171:         return TOK_TRUE;
dvander@171:     if (!strcmp(str,  "false"))
dvander@171:         return TOK_FALSE;
dvander@171:     if (!strcmp(str,  "struct"))
dvander@171:         return TOK_STRUCT;
dvander@171:     if (!strcmp(str,  "switch"))
dvander@171:         return TOK_SWITCH;
dvander@171:     if (!strcmp(str,  "case"))
dvander@171:         return TOK_CASE;
dvander@171:     if (!strcmp(str,  "default"))
dvander@171:         return TOK_DEFAULT;
dvander@174:     if (!strcmp(str,  "typedef"))
dvander@174:         return TOK_TYPEDEF;
dvander@174:     if (!strcmp(str,  "void"))
dvander@174:         return TOK_VOID;
dvander@174:     if (!strcmp(str,  "int"))
dvander@174:         return TOK_INT;
dvander@194:     if (!strcmp(str,  "sizeof"))
dvander@194:         return TOK_SIZEOF;
dvander@195:     if (!strcmp(str,  "import"))
dvander@195:         return TOK_IMPORT;
dvander@171:     return TOK_NONE;
dvander@171: }
dvander@171: 
dvander@0: TokenKind
dvander@0: Scanner::identifier(char first)
dvander@0: {
danderson@92:     literal_.clear();
danderson@92:     if (!literal_.append(first))
danderson@92:         return TOK_ERROR;
dvander@0: 
dvander@4:     char c;
dvander@0:     for (;;) {
dvander@4:         c = getChar();
dvander@171:         if (!isascii(c) || (!isalnum(c) && c != '_'))
dvander@0:             break;
danderson@92:         if (!literal_.append(c))
danderson@92:             return TOK_ERROR;
dvander@0:     }
dvander@133:     
danderson@92:     if (!literal_.append('\0'))
danderson@92:         return TOK_ERROR;
danderson@92: 
dvander@171:     // Most keywords do not act as labels.
dvander@171:     TokenKind kind = FindIdentifier(literal());
dvander@171:     if (kind != TOK_NONE) {
dvander@171:         putChar(c);
dvander@171:         return kind;
dvander@171:     }
dvander@171: 
dvander@172:     if (c == ':' && allowTags_)
dvander@171:         return TOK_LABEL;
dvander@171: 
dvander@171:     putChar(c);
dvander@171: 
dvander@171:     // _ is a special case which acts as a tag and a keyword.
dvander@171:     if (!strcmp(literal(), "_"))
dvander@133:         return TOK_UNDERSCORE;
dvander@171: 
dvander@171:     return TOK_NAME;
dvander@0: }
dvander@0: 
dvander@0: TokenKind
dvander@0: Scanner::next()
dvander@0: {
dvander@0:     assert(!pushed_);
dvander@0: 
dvander@0:     for (;;) {
dvander@25:         begin_ = cur_;
dvander@25: 
dvander@0:         char c = getChar();
dvander@0:         switch (c) {
dvander@0:           case '/':
dvander@0:           {
dvander@0:             c = getChar();
dvander@0:             if (c == '/') {
dvander@0:                 eatSingleLine();
dvander@0:                 break;
dvander@0:             }
dvander@0:             if (c == '*') {
dvander@25:                 if (!eatComment()) {
dvander@25:                     cc_.reportError(begin(), Message_UnterminatedComment);
dvander@0:                     return TOK_ERROR;
dvander@25:                 }
dvander@0:                 break;
dvander@0:             }
dvander@0:             putChar(c);
dvander@0:             return TOK_SLASH;
dvander@0:           }
dvander@0: 
dvander@25:           case '\n':
dvander@25:           case '\r':
dvander@25:             calcNewLine(c);
dvander@25:             break;
dvander@25: 
dvander@0:           case ';': return TOK_SEMICOLON;
dvander@0:           case '{': return TOK_LBRACE;
dvander@0:           case '}': return TOK_RBRACE;
dvander@0:           case '(': return TOK_LPAREN;
dvander@0:           case ')': return TOK_RPAREN;
dvander@0:           case '[': return TOK_LBRACKET;
dvander@0:           case ']': return TOK_RBRACKET;
dvander@78:           case '~': return TOK_TILDE;
dvander@83:           case '?': return TOK_QMARK;
dvander@83:           case ':': return TOK_COLON;
dvander@2: 
dvander@68:           case '.':
dvander@68:             c = getChar();
dvander@68:             if (c == '.') {
dvander@68:                 c = getChar();
dvander@68:                 if (c == '.')
dvander@68:                     return TOK_ELLIPSES;
dvander@148:                 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
dvander@148:                 return TOK_ERROR;
dvander@68:             }
dvander@148:             putChar(c);
dvander@148:             return TOK_DOT;
dvander@68: 
dvander@2:           case '*':
dvander@2:             c = getChar();
dvander@2:             if (c == '=')
dvander@2:                 return TOK_ASSIGN_MUL;
dvander@2:             putChar(c);
dvander@2:             return TOK_STAR;
dvander@2: 
dvander@2:           case '+':
dvander@2:             c = getChar();
dvander@2:             if (c == '=')
dvander@2:                 return TOK_ASSIGN_ADD;
dvander@23:             if (c == '+')
dvander@23:                 return TOK_INCREMENT;
dvander@2:             putChar(c);
dvander@2:             return TOK_PLUS;
dvander@2: 
dvander@8:           case '&':
dvander@22:             c = getChar();
dvander@22:             if (c == '=')
dvander@22:                 return TOK_ASSIGN_BITAND;
dvander@75:             if (c == '&')
dvander@75:                 return TOK_AND;
dvander@22:             putChar(c);
dvander@8:             return TOK_BITAND;
dvander@8: 
dvander@8:           case '|':
dvander@22:             c = getChar();
dvander@22:             if (c == '=')
dvander@22:                 return TOK_ASSIGN_BITOR;
dvander@75:             if (c == '|')
dvander@75:                 return TOK_OR;
dvander@22:             putChar(c);
dvander@8:             return TOK_BITOR;
dvander@8: 
dvander@8:           case '^':
dvander@22:             c = getChar();
dvander@22:             if (c == '=')
dvander@22:                 return TOK_ASSIGN_BITXOR;
dvander@22:             putChar(c);
dvander@8:             return TOK_BITXOR;
dvander@8: 
dvander@82:           case '%':
dvander@82:             c = getChar();
dvander@82:             if (c == '=')
dvander@82:                 return TOK_ASSIGN_MOD;
dvander@82:             putChar(c);
dvander@82:             return TOK_PERCENT;
dvander@82: 
dvander@22:           case '-':
dvander@22:             c = getChar();
dvander@22:             if (c == '=')
dvander@22:                 return TOK_ASSIGN_SUB;
dvander@23:             if (c == '-')
dvander@23:                 return TOK_DECREMENT;
dvander@22:             putChar(c);
dvander@22:             return TOK_MINUS;
dvander@22: 
dvander@0:           case ',': return TOK_COMMA;
dvander@0:           case '\0': return TOK_EOF;
dvander@0: 
dvander@74:           case '!':
dvander@74:             c = getChar();
dvander@74:             if (c == '=')
dvander@74:                 return TOK_NOTEQUALS;
dvander@74:             putChar(c);
dvander@74:             return TOK_NOT;
dvander@74: 
dvander@0:           case '=':
dvander@0:             c = getChar();
dvander@0:             if (c == '=')
dvander@0:                 return TOK_EQUALS;
dvander@0:             putChar(c);
dvander@0:             return TOK_ASSIGN;
dvander@0: 
dvander@148: 
dvander@0:           case '<':
dvander@0:             c = getChar();
dvander@0:             if (c == '=')
dvander@0:                 return TOK_LE;
dvander@2:             if (c == '<') {
dvander@2:                 c = getChar();
dvander@2:                 if (c == '=')
dvander@2:                     return TOK_ASSIGN_SHL;
dvander@138:                 putChar(c);
dvander@2:                 return TOK_SHL;
dvander@2:             }
dvander@0:             putChar(c);
dvander@0:             return TOK_LT;
dvander@0: 
dvander@0:           case '>':
dvander@0:             c = getChar();
dvander@0:             if (c == '=')
dvander@0:                 return TOK_GE;
dvander@8:             if (c == '>') {
dvander@8:                 c = getChar();
dvander@22:                 if (c == '>') {
dvander@22:                     c = getChar();
dvander@22:                     if (c == '=')
dvander@22:                         return TOK_ASSIGN_USHR;
dvander@22:                     putChar(c);
dvander@8:                     return TOK_USHR;
dvander@22:                 }
dvander@8:                 return TOK_SHR;
dvander@8:             }
dvander@0:             putChar(c);
dvander@0:             return TOK_GT;
dvander@0: 
dvander@0:           case ' ':
dvander@0:           case '\t':
dvander@0:           case '\f':
dvander@0:             break;
dvander@0: 
dvander@21:           case '"':
dvander@21:             return stringLiteral();
dvander@21: 
dvander@0:           default:
dvander@0:             if (c >= '0' && c <= '9')
dvander@4:                 return numberLiteral(c);
dvander@0:             if (c == '_' || (isascii(c) && isalpha(c)))
dvander@0:                 return identifier(c);
dvander@25:             cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
dvander@0:             return TOK_ERROR;
dvander@0:         }
dvander@0:     }
dvander@0: }
dvander@0: 
dvander@0: TokenKind
dvander@0: Scanner::scan()
dvander@0: {
dvander@0:     if (pushed_) {
dvander@0:         pushed_ = false;
dvander@0:         return lastToken_;
dvander@0:     }
dvander@0:     lastToken_ = next();
dvander@0:     return lastToken_;
dvander@0: }
dvander@0: 
dvander@0: TokenKind
dvander@0: Scanner::peek()
dvander@0: {
dvander@0:     if (pushed_)
dvander@0:         return lastToken_;
dvander@0:     scan();
dvander@0:     pushed_ = true;
dvander@0:     return lastToken_;
dvander@0: }