asherkin@263: /* vim: set ts=4 sw=4 tw=99 et:
asherkin@263:  *
asherkin@263:  * Copyright (C) 2012 David Anderson
asherkin@263:  *
asherkin@263:  * This file is part of JITCraft.
asherkin@263:  *
asherkin@263:  * JITCraft is free software: you can redistribute it and/or modify it under
asherkin@263:  * the terms of the GNU General Public License as published by the Free
asherkin@263:  * Software Foundation, either version 3 of the License, or (at your option)
asherkin@263:  * any later version.
asherkin@263:  * 
asherkin@263:  * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
asherkin@263:  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
asherkin@263:  * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
asherkin@263:  *
asherkin@263:  * You should have received a copy of the GNU General Public License along with
asherkin@263:  * JITCraft. If not, see http://www.gnu.org/licenses/.
asherkin@263:  */
asherkin@263: #include <assert.h>
asherkin@263: #include <ctype.h>
asherkin@263: #include <math.h>
asherkin@263: #include <limits.h>
asherkin@263: #include <string.h>
asherkin@263: #include "../Utility.h"
asherkin@263: #include "../Heap-inl.h"
asherkin@263: #include "../Handles-inl.h"
asherkin@263: #include "Scanner.h"
asherkin@263: 
asherkin@263: using namespace ke;
asherkin@263: 
asherkin@263: const char *ke::TokenNames[] =
asherkin@263: {
asherkin@263: #define _(name, str) str,
asherkin@263:     TOKENMAP(_)
asherkin@263: #undef _
asherkin@263:     NULL
asherkin@263: };
asherkin@263: 
asherkin@263: Scanner::Scanner(CompileContext &cc, const char *stream, size_t length)
asherkin@263:   : cc_(cc),
asherkin@263:     stream_(stream),
asherkin@263:     length_(length),
asherkin@263:     pushed_(false),
asherkin@263:     allowTags_(true),
asherkin@263:     saved_('\0'),
asherkin@263:     pos_(stream_)
asherkin@263: {
asherkin@263:     cur_.line = 1;
asherkin@263:     cur_.col = 1;
asherkin@263:     last_ = cur_;
asherkin@263: }
asherkin@263: 
asherkin@263: char
asherkin@263: Scanner::getChar()
asherkin@263: {
asherkin@263:     if (saved_ != '\0') {
asherkin@263:         char temp = saved_;
asherkin@263:         saved_ = '\0';
asherkin@263:         return temp;
asherkin@263:     }
asherkin@263:     if (pos_ >= stream_ + length_)
asherkin@263:         return '\0';
asherkin@263:     last_.col = cur_.col++;
asherkin@263:     return *pos_++;
asherkin@263: }
asherkin@263: 
asherkin@263: void
asherkin@263: Scanner::putChar(char c)
asherkin@263: {
asherkin@263:     assert(saved_ == '\0');
asherkin@263:     saved_ = c;
asherkin@263:     cur_.col = last_.col;
asherkin@263: }
asherkin@263: 
asherkin@263: void
asherkin@263: Scanner::calcNewLine(char c)
asherkin@263: {
asherkin@263:     cur_.line++;
asherkin@263:     cur_.col = 1;
asherkin@263:     last_ = cur_;
asherkin@263: 
asherkin@263:     char o;
asherkin@263:     if (c == '\r' && ((o = getChar()) != '\n'))
asherkin@263:         putChar(o);
asherkin@263: }
asherkin@263: 
asherkin@263: void
asherkin@263: Scanner::eatSingleLine()
asherkin@263: {
asherkin@263:     char c;
asherkin@263:     while ((c = getChar()) != '\0') {
asherkin@263:         if (c == '\n' || c == '\r') {
asherkin@263:             calcNewLine(c);
asherkin@263:             break;
asherkin@263:         }
asherkin@263:     }
asherkin@263: }
asherkin@263: 
asherkin@263: bool
asherkin@263: Scanner::eatComment()
asherkin@263: {
asherkin@263:     char c;
asherkin@263: 
asherkin@263:     while ((c = getChar()) != '\0') {
asherkin@263:         while (c == '*') {
asherkin@263:             c = getChar();
asherkin@263:             if (c == '\0')
asherkin@263:                 return false;
asherkin@263:             if (c == '/')
asherkin@263:                 return true;
asherkin@263:         }
asherkin@263:         if (c == '\r' || c == '\n')
asherkin@263:             calcNewLine(c);
asherkin@263:     }
asherkin@263: 
asherkin@263:     return false;
asherkin@263: }
asherkin@263: 
asherkin@263: static inline bool
asherkin@263: IsDigit(char c)
asherkin@263: {
asherkin@263:     return c >= '0' && c <= '9';
asherkin@263: }
asherkin@263: 
asherkin@263: // Based off the logic in sc2.c's ftoi()...
asherkin@263: double
asherkin@263: Scanner::ParseDouble(const char *string)
asherkin@263: {
asherkin@263:     const char *ptr = string;
asherkin@263: 
asherkin@263:     double number = 0.0;
asherkin@263:     while (IsDigit(*ptr)) {
asherkin@263:         number = (number * 10) + (*ptr - '0');
asherkin@263:         ptr++;
asherkin@263:     }
asherkin@263: 
asherkin@263:     assert(*ptr == '.');
asherkin@263:     ptr++;
asherkin@263: 
asherkin@263:     double fraction = 0.0;
asherkin@263:     double multiplier = 1.0;
asherkin@263:     while (IsDigit(*ptr)) {
asherkin@263:         fraction = (fraction * 10) + (*ptr - '0');
asherkin@263:         multiplier = multiplier / 10.0;
asherkin@263:         ptr++;
asherkin@263:     }
asherkin@263: 
asherkin@263:     number += fraction * multiplier;
asherkin@263: 
asherkin@263:     if (*ptr++ == 'e') {
asherkin@263:         int sign = 1;
asherkin@263:         if (*ptr == '-') {
asherkin@263:             sign = -1;
asherkin@263:             ptr++;
asherkin@263:         }
asherkin@263: 
asherkin@263:         int exponent = 0;
asherkin@263:         while (IsDigit(*ptr)) {
asherkin@263:             exponent = (exponent * 10) + (*ptr - '0');
asherkin@263:             ptr++;
asherkin@263:         }
asherkin@263: 
asherkin@263:         multiplier = pow(10.0, exponent * sign);
asherkin@263:         number *= multiplier;
asherkin@263:     }
asherkin@263: 
asherkin@263:     return number;
asherkin@263: }
asherkin@263: 
asherkin@263: static inline bool
asherkin@263: IsHexDigit(char c)
asherkin@263: {
asherkin@263:     return IsDigit(c) ||
asherkin@263:            (c >= 'a' && c <= 'f') ||
asherkin@263:            (c >= 'A' && c <= 'F');
asherkin@263: }
asherkin@263: 
asherkin@263: // Based on the logic for litchar() in sc2.c.
asherkin@263: int
asherkin@263: Scanner::readEscapeCode()
asherkin@263: {
asherkin@263:     char c = getChar();
asherkin@263:     if (c == '\\')
asherkin@263:         return c;
asherkin@263: 
asherkin@263:     switch (c) {
asherkin@263:       case 'a':
asherkin@263:         return '\a';
asherkin@263:       case 'b':
asherkin@263:         return '\b';
asherkin@263:       case 'e':
asherkin@263:         return 27;          // Apparently \e is non-standard.
asherkin@263:       case 'f':
asherkin@263:         return '\f';
asherkin@263:       case 'n':
asherkin@263:         return '\n';
asherkin@263:       case 'r':
asherkin@263:         return '\r';
asherkin@263:       case 't':
asherkin@263:         return '\t';
asherkin@263:       case 'v':
asherkin@263:         return '\v';
asherkin@263: 
asherkin@263:       case 'x':
asherkin@263:       {
asherkin@263:           unsigned digits = 0;
asherkin@263:           char r = 0;
asherkin@263:   
asherkin@263:           c = getChar();
asherkin@263:           while (IsHexDigit(c) && digits < 2) {
asherkin@263:               if (IsDigit(c))
asherkin@263:                   c = (c << 4) + (c - '0');
asherkin@263:               else
asherkin@263:                   c = (c << 4) + (tolower(c) - 'a' + 10);
asherkin@263:               digits++;
asherkin@263:               c = getChar();
asherkin@263:            }
asherkin@263:                 
asherkin@263:           // Swallow a trailing ';'
asherkin@263:           if (c != ';')
asherkin@263:               putChar(c);
asherkin@263:   
asherkin@263:           return r;
asherkin@263:       }
asherkin@263: 
asherkin@263:       case '\'':
asherkin@263:       case '\"':
asherkin@263:       case '%':
asherkin@263:         return c;
asherkin@263: 
asherkin@263:       default:
asherkin@263:       {
asherkin@263:         if (IsDigit(c)) {
asherkin@263:             // \ddd
asherkin@263:             char r = 0;
asherkin@263:             while (IsDigit(c)) {
asherkin@263:                 r = r * 10 + (c - '0');
asherkin@263:                 c = getChar();
asherkin@263:             }
asherkin@263: 
asherkin@263:             // Swallow a trailing ;
asherkin@263:             if (c != ';')
asherkin@263:                 putChar(c);
asherkin@263: 
asherkin@263:             return r;
asherkin@263:         }
asherkin@263:         cc_.reportError(begin(), Message_UnknownEscapeCode, c);
asherkin@263:         return INT_MAX;
asherkin@263:       }
asherkin@263:     }
asherkin@263: }
asherkin@263: 
asherkin@263: TokenKind
asherkin@263: Scanner::stringLiteral()
asherkin@263: {
asherkin@263:     literal_.clear();
asherkin@263: 
asherkin@263:     for (;;) {
asherkin@263:         char c = getChar();
asherkin@263:         if (c == '\"')
asherkin@263:             break;
asherkin@263:         if (c == '\r' || c == '\n' || c == '\0') {
asherkin@263:             cc_.reportError(begin(), Message_UnterminatedString);
asherkin@263:             return TOK_ERROR;
asherkin@263:         }
asherkin@263:         if (c == '\\') {
asherkin@263:             int code = readEscapeCode();
asherkin@263:             if (code == INT_MAX)
asherkin@263:                 return TOK_ERROR;
asherkin@263:             c = char(code);
asherkin@263:         }
asherkin@263:         if (!literal_.append(c))
asherkin@263:             return TOK_ERROR;
asherkin@263:     }
asherkin@263: 
asherkin@263:     if (!literal_.append('\0'))
asherkin@263:         return TOK_ERROR;
asherkin@263: 
asherkin@263:     return TOK_STRING_LITERAL;
asherkin@263: }
asherkin@263: 
asherkin@263: TokenKind
asherkin@263: Scanner::hexLiteral()
asherkin@263: {
asherkin@263:     literal_.clear();
asherkin@263: 
asherkin@263:     char c;
asherkin@263:     for (;;) {
asherkin@263:         c = getChar();
asherkin@263:         if (!IsHexDigit(c))
asherkin@263:             break;
asherkin@263:         if (!literal_.append(c))
asherkin@263:             return TOK_ERROR;
asherkin@263:     }
asherkin@263: 
asherkin@263:     putChar(c);
asherkin@263: 
asherkin@263:     if (!literal_.append('\0'))
asherkin@263:         return TOK_ERROR;
asherkin@263:     return TOK_HEX_LITERAL;
asherkin@263: }
asherkin@263: 
asherkin@263: TokenKind
asherkin@263: Scanner::numberLiteral(char first)
asherkin@263: {
asherkin@263:     literal_.clear();
asherkin@263:     if (!literal_.append(first))
asherkin@263:         return TOK_ERROR;
asherkin@263: 
asherkin@263:     char c;
asherkin@263:     for (;;) {
asherkin@263:         c = getChar();
asherkin@263:         if (!IsDigit(c))
asherkin@263:             break;
asherkin@263:         if (!literal_.append(c))
asherkin@263:             return TOK_ERROR;
asherkin@263:     }
asherkin@263: 
asherkin@263:     // Detect a hexadecimal string.
asherkin@263:     if (literal_.length() == 1 &&
asherkin@263:         literal_[0] == '0' &&
asherkin@263:         (c == 'x' || c == 'X'))
asherkin@263:     {
asherkin@263:         return hexLiteral();
asherkin@263:     }
asherkin@263: 
asherkin@263:     if (c != '.') {
asherkin@263:         putChar(c);
asherkin@263:         if (!literal_.append('\0'))
asherkin@263:             return TOK_ERROR;
asherkin@263:         return TOK_INTEGER_LITERAL;
asherkin@263:     }
asherkin@263:     if (!literal_.append(c))
asherkin@263:         return TOK_ERROR;
asherkin@263: 
asherkin@263:     c = getChar();
asherkin@263:     if (!IsDigit(c)) {
asherkin@263:         cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
asherkin@263:         return TOK_ERROR;
asherkin@263:     }
asherkin@263:     if (!literal_.append(c))
asherkin@263:         return TOK_ERROR;
asherkin@263: 
asherkin@263:     for (;;) {
asherkin@263:         c = getChar();
asherkin@263:         if (!IsDigit(c))
asherkin@263:             break;
asherkin@263:         if (!literal_.append(c))
asherkin@263:             return TOK_ERROR;
asherkin@263:     }
asherkin@263: 
asherkin@263:     if (c != 'e') {
asherkin@263:         putChar(c);
asherkin@263:     } else {
asherkin@263:         if (!literal_.append(c))
asherkin@263:             return TOK_ERROR;
asherkin@263:         c = getChar();
asherkin@263:         if (c == '-') {
asherkin@263:             if (!literal_.append(c))
asherkin@263:                 return TOK_ERROR;
asherkin@263:             c = getChar();
asherkin@263:         }
asherkin@263:         if (!IsDigit(c)) {
asherkin@263:             cc_.reportError(begin(), Message_ExpectedDigitForFloat, c);
asherkin@263:             return TOK_ERROR;
asherkin@263:         }
asherkin@263:         if (!literal_.append(c))
asherkin@263:             return TOK_ERROR;
asherkin@263:         for (;;) {
asherkin@263:             c = getChar();
asherkin@263:             if (!IsDigit(c)) {
asherkin@263:                 putChar(c);
asherkin@263:                 break;
asherkin@263:             }
asherkin@263:             if (!literal_.append(c))
asherkin@263:                 return TOK_ERROR;
asherkin@263:         }
asherkin@263:     }
asherkin@263: 
asherkin@263:     if (!literal_.append('\0'))
asherkin@263:         return TOK_ERROR;
asherkin@263: 
asherkin@263:     return TOK_FLOAT_LITERAL;
asherkin@263: }
asherkin@263: 
asherkin@263: static inline TokenKind
asherkin@263: FindIdentifier(const char *str)
asherkin@263: {
asherkin@263:     if (!strcmp(str,  "new"))
asherkin@263:         return TOK_NEW;
asherkin@263:     if (!strcmp(str,  "for"))
asherkin@263:         return TOK_FOR;
asherkin@263:     if (!strcmp(str,  "return"))
asherkin@263:         return TOK_RETURN;
asherkin@263:     if (!strcmp(str,  "if"))
asherkin@263:         return TOK_IF;
asherkin@263:     if (!strcmp(str,  "else"))
asherkin@263:         return TOK_ELSE;
asherkin@263:     if (!strcmp(str,  "enum"))
asherkin@263:         return TOK_ENUM;
asherkin@263:     if (!strcmp(str,  "while"))
asherkin@263:         return TOK_WHILE;
asherkin@263:     if (!strcmp(str,  "do"))
asherkin@263:         return TOK_DO;
asherkin@263:     if (!strcmp(str,  "break"))
asherkin@263:         return TOK_BREAK;
asherkin@263:     if (!strcmp(str,  "continue"))
asherkin@263:         return TOK_CONTINUE;
asherkin@263:     if (!strcmp(str,  "forward"))
asherkin@263:         return TOK_FORWARD;
asherkin@263:     if (!strcmp(str,  "public"))
asherkin@263:         return TOK_PUBLIC;
asherkin@263:     if (!strcmp(str,  "const"))
asherkin@263:         return TOK_CONST;
asherkin@263:     if (!strcmp(str,  "native"))
asherkin@263:         return TOK_NATIVE;
asherkin@263:     if (!strcmp(str,  "stock"))
asherkin@263:         return TOK_STOCK;
asherkin@263:     if (!strcmp(str,  "true"))
asherkin@263:         return TOK_TRUE;
asherkin@263:     if (!strcmp(str,  "false"))
asherkin@263:         return TOK_FALSE;
asherkin@263:     if (!strcmp(str,  "struct"))
asherkin@263:         return TOK_STRUCT;
asherkin@263:     if (!strcmp(str,  "switch"))
asherkin@263:         return TOK_SWITCH;
asherkin@263:     if (!strcmp(str,  "case"))
asherkin@263:         return TOK_CASE;
asherkin@263:     if (!strcmp(str,  "default"))
asherkin@263:         return TOK_DEFAULT;
asherkin@263:     if (!strcmp(str,  "typedef"))
asherkin@263:         return TOK_TYPEDEF;
asherkin@263:     if (!strcmp(str,  "void"))
asherkin@263:         return TOK_VOID;
asherkin@263:     if (!strcmp(str,  "int"))
asherkin@263:         return TOK_INT;
asherkin@263:     if (!strcmp(str,  "sizeof"))
asherkin@263:         return TOK_SIZEOF;
asherkin@263:     if (!strcmp(str,  "import"))
asherkin@263:         return TOK_IMPORT;
asherkin@263:     if (!strcmp(str,  "from"))
asherkin@263:         return TOK_FROM;
asherkin@263:     return TOK_NONE;
asherkin@263: }
asherkin@263: 
asherkin@263: TokenKind
asherkin@263: Scanner::identifier(char first)
asherkin@263: {
asherkin@263:     literal_.clear();
asherkin@263:     if (!literal_.append(first))
asherkin@263:         return TOK_ERROR;
asherkin@263: 
asherkin@263:     char c;
asherkin@263:     for (;;) {
asherkin@263:         c = getChar();
asherkin@263:         if (!isascii(c) || (!isalnum(c) && c != '_'))
asherkin@263:             break;
asherkin@263:         if (!literal_.append(c))
asherkin@263:             return TOK_ERROR;
asherkin@263:     }
asherkin@263:     
asherkin@263:     if (!literal_.append('\0'))
asherkin@263:         return TOK_ERROR;
asherkin@263: 
asherkin@263:     // Most keywords do not act as labels.
asherkin@263:     TokenKind kind = FindIdentifier(literal());
asherkin@263:     if (kind != TOK_NONE) {
asherkin@263:         putChar(c);
asherkin@263:         return kind;
asherkin@263:     }
asherkin@263: 
asherkin@263:     if (c == ':' && allowTags_)
asherkin@263:         return TOK_LABEL;
asherkin@263: 
asherkin@263:     putChar(c);
asherkin@263: 
asherkin@263:     // _ is a special case which acts as a tag and a keyword.
asherkin@263:     if (!strcmp(literal(), "_"))
asherkin@263:         return TOK_UNDERSCORE;
asherkin@263: 
asherkin@263:     return TOK_NAME;
asherkin@263: }
asherkin@263: 
asherkin@263: TokenKind
asherkin@263: Scanner::next()
asherkin@263: {
asherkin@263:     assert(!pushed_);
asherkin@263: 
asherkin@263:     for (;;) {
asherkin@263:         begin_ = cur_;
asherkin@263: 
asherkin@263:         char c = getChar();
asherkin@263:         switch (c) {
asherkin@263:           case '/':
asherkin@263:           {
asherkin@263:             c = getChar();
asherkin@263:             if (c == '/') {
asherkin@263:                 eatSingleLine();
asherkin@263:                 break;
asherkin@263:             }
asherkin@263:             if (c == '*') {
asherkin@263:                 if (!eatComment()) {
asherkin@263:                     cc_.reportError(begin(), Message_UnterminatedComment);
asherkin@263:                     return TOK_ERROR;
asherkin@263:                 }
asherkin@263:                 break;
asherkin@263:             }
asherkin@263:             putChar(c);
asherkin@263:             return TOK_SLASH;
asherkin@263:           }
asherkin@263: 
asherkin@263:           case '\n':
asherkin@263:           case '\r':
asherkin@263:             calcNewLine(c);
asherkin@263:             break;
asherkin@263: 
asherkin@263:           case ';': return TOK_SEMICOLON;
asherkin@263:           case '{': return TOK_LBRACE;
asherkin@263:           case '}': return TOK_RBRACE;
asherkin@263:           case '(': return TOK_LPAREN;
asherkin@263:           case ')': return TOK_RPAREN;
asherkin@263:           case '[': return TOK_LBRACKET;
asherkin@263:           case ']': return TOK_RBRACKET;
asherkin@263:           case '~': return TOK_TILDE;
asherkin@263:           case '?': return TOK_QMARK;
asherkin@263:           case ':': return TOK_COLON;
asherkin@263: 
asherkin@263:           case '.':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '.') {
asherkin@263:                 c = getChar();
asherkin@263:                 if (c == '.')
asherkin@263:                     return TOK_ELLIPSES;
asherkin@263:                 cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
asherkin@263:                 return TOK_ERROR;
asherkin@263:             }
asherkin@263:             putChar(c);
asherkin@263:             return TOK_DOT;
asherkin@263: 
asherkin@263:           case '*':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_ASSIGN_MUL;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_STAR;
asherkin@263: 
asherkin@263:           case '+':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_ASSIGN_ADD;
asherkin@263:             if (c == '+')
asherkin@263:                 return TOK_INCREMENT;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_PLUS;
asherkin@263: 
asherkin@263:           case '&':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_ASSIGN_BITAND;
asherkin@263:             if (c == '&')
asherkin@263:                 return TOK_AND;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_BITAND;
asherkin@263: 
asherkin@263:           case '|':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_ASSIGN_BITOR;
asherkin@263:             if (c == '|')
asherkin@263:                 return TOK_OR;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_BITOR;
asherkin@263: 
asherkin@263:           case '^':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_ASSIGN_BITXOR;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_BITXOR;
asherkin@263: 
asherkin@263:           case '%':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_ASSIGN_MOD;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_PERCENT;
asherkin@263: 
asherkin@263:           case '-':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_ASSIGN_SUB;
asherkin@263:             if (c == '-')
asherkin@263:                 return TOK_DECREMENT;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_MINUS;
asherkin@263: 
asherkin@263:           case ',': return TOK_COMMA;
asherkin@263:           case '\0': return TOK_EOF;
asherkin@263: 
asherkin@263:           case '!':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_NOTEQUALS;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_NOT;
asherkin@263: 
asherkin@263:           case '=':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_EQUALS;
asherkin@263:             putChar(c);
asherkin@263:             return TOK_ASSIGN;
asherkin@263: 
asherkin@263: 
asherkin@263:           case '<':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_LE;
asherkin@263:             if (c == '<') {
asherkin@263:                 c = getChar();
asherkin@263:                 if (c == '=')
asherkin@263:                     return TOK_ASSIGN_SHL;
asherkin@263:                 putChar(c);
asherkin@263:                 return TOK_SHL;
asherkin@263:             }
asherkin@263:             putChar(c);
asherkin@263:             return TOK_LT;
asherkin@263: 
asherkin@263:           case '>':
asherkin@263:             c = getChar();
asherkin@263:             if (c == '=')
asherkin@263:                 return TOK_GE;
asherkin@263:             if (c == '>') {
asherkin@263:                 c = getChar();
asherkin@263:                 if (c == '>') {
asherkin@263:                     c = getChar();
asherkin@263:                     if (c == '=')
asherkin@263:                         return TOK_ASSIGN_USHR;
asherkin@263:                     putChar(c);
asherkin@263:                     return TOK_USHR;
asherkin@263:                 }
asherkin@263:                 return TOK_SHR;
asherkin@263:             }
asherkin@263:             putChar(c);
asherkin@263:             return TOK_GT;
asherkin@263: 
asherkin@263:           case ' ':
asherkin@263:           case '\t':
asherkin@263:           case '\f':
asherkin@263:             break;
asherkin@263: 
asherkin@263:           case '"':
asherkin@263:             return stringLiteral();
asherkin@263: 
asherkin@263:           default:
asherkin@263:             if (c >= '0' && c <= '9')
asherkin@263:                 return numberLiteral(c);
asherkin@263:             if (c == '_' || (isascii(c) && isalpha(c)))
asherkin@263:                 return identifier(c);
asherkin@263:             cc_.reportError(begin_, Message_UnexpectedCharacter, c, uint8(c));
asherkin@263:             return TOK_ERROR;
asherkin@263:         }
asherkin@263:     }
asherkin@263: }
asherkin@263: 
asherkin@263: TokenKind
asherkin@263: Scanner::scan()
asherkin@263: {
asherkin@263:     if (pushed_) {
asherkin@263:         pushed_ = false;
asherkin@263:         return lastToken_;
asherkin@263:     }
asherkin@263:     lastToken_ = next();
asherkin@263:     return lastToken_;
asherkin@263: }
asherkin@263: 
asherkin@263: TokenKind
asherkin@263: Scanner::peek()
asherkin@263: {
asherkin@263:     if (pushed_)
asherkin@263:         return lastToken_;
asherkin@263:     scan();
asherkin@263:     pushed_ = true;
asherkin@263:     return lastToken_;
asherkin@263: }