dvander@0: /*
dvander@0:  * Copyright (C) 2012 David Anderson
dvander@0:  *
dvander@0:  * This file is part of JITCraft.
dvander@0:  *
dvander@0:  * JITCraft is free software: you can redistribute it and/or modify it under
dvander@0:  * the terms of the GNU General Public License as published by the Free
dvander@0:  * Software Foundation, either version 3 of the License, or (at your option)
dvander@0:  * any later version.
dvander@0:  * 
dvander@0:  * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
dvander@0:  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
dvander@0:  * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
dvander@0:  *
dvander@0:  * You should have received a copy of the GNU General Public License along with
dvander@0:  * JITCraft. If not, see http://www.gnu.org/licenses/.
dvander@0:  */
dvander@0: #include <assert.h>
dvander@0: #include <ctype.h>
dvander@0: #include "Scanner.h"
dvander@0: 
dvander@0: using namespace jc;
dvander@0: 
dvander@0: const char *jc::TokenNames[] =
dvander@0: {
dvander@0: #define _(name, str) str,
dvander@0:     TOKENMAP(_)
dvander@0: #undef _
dvander@0:     NULL
dvander@0: };
dvander@0: 
dvander@0: Scanner::Scanner(const char *stream, size_t length)
dvander@0:   : stream_(stream),
dvander@0:     length_(length),
dvander@0:     pushed_(false),
dvander@0:     saved_('\0'),
dvander@0:     pos_(stream_)
dvander@0: {
dvander@0: }
dvander@0: 
dvander@0: char
dvander@0: Scanner::getChar()
dvander@0: {
dvander@0:     if (saved_ != '\0') {
dvander@0:         char temp = saved_;
dvander@0:         saved_ = '\0';
dvander@0:         return temp;
dvander@0:     }
dvander@0:     if (pos_ >= stream_ + length_)
dvander@0:         return '\0';
dvander@0:     return *pos_++;
dvander@0: }
dvander@0: 
dvander@0: void
dvander@0: Scanner::putChar(char c)
dvander@0: {
dvander@0:     assert(saved_ == '\0');
dvander@0:     saved_ = c;
dvander@0: }
dvander@0: 
dvander@0: void
dvander@0: Scanner::eatSingleLine()
dvander@0: {
dvander@0:     char c;
dvander@0:     while ((c = getChar()) != '\0') {
dvander@0:         if (c == '\n' || c == '\r')
dvander@0:             break;
dvander@0:     }
dvander@0: }
dvander@0: 
dvander@0: bool
dvander@0: Scanner::eatComment()
dvander@0: {
dvander@0:     char c;
dvander@0: 
dvander@0:     while ((c = getChar()) != '\0') {
dvander@0:         while (c == '*') {
dvander@0:             c = getChar();
dvander@0:             if (c == '\0')
dvander@0:                 return false;
dvander@0:             if (c == '/')
dvander@0:                 return true;
dvander@0:         }
dvander@0:     }
dvander@0: 
dvander@0:     return false;
dvander@0: }
dvander@0: 
dvander@4: static inline bool
dvander@4: IsDigit(char c)
dvander@4: {
dvander@4:     return c >= '0' && c <= '9';
dvander@4: }
dvander@4: 
dvander@4: // Based off the logic in sc2.c's ftoi()...
dvander@4: double
dvander@4: Scanner::ParseDouble(const char *string)
dvander@4: {
dvander@4:     const char *ptr = string;
dvander@4: 
dvander@4:     double number = 0.0;
dvander@4:     while (IsDigit(*ptr)) {
dvander@4:         number = (number * 10) + (*ptr - '0');
dvander@4:         ptr++;
dvander@4:     }
dvander@4: 
dvander@4:     assert(*ptr == '.');
dvander@4:     ptr++;
dvander@4: 
dvander@4:     double fraction = 0.0;
dvander@4:     double multiplier = 1.0;
dvander@4:     while (IsDigit(*ptr)) {
dvander@4:         fraction = (fraction * 10) + (*ptr - '0');
dvander@4:         multiplier = multiplier / 10.0;
dvander@4:         ptr++;
dvander@4:     }
dvander@4: 
dvander@4:     number += fraction * multiplier;
dvander@4: 
dvander@4:     if (*ptr++ == 'e') {
dvander@4:         int sign = 1;
dvander@4:         if (*ptr == '-') {
dvander@4:             sign = -1;
dvander@4:             ptr++;
dvander@4:         }
dvander@4: 
dvander@4:         int exponent = 0;
dvander@4:         while (IsDigit(*ptr)) {
dvander@4:             exponent = (exponent * 10) + (*ptr - '0');
dvander@4:             ptr++;
dvander@4:         }
dvander@4: 
dvander@4:         multiplier = pow(10.0, exponent * sign);
dvander@4:         number *= multiplier;
dvander@4:     }
dvander@4: 
dvander@4:     return number;
dvander@4: }
dvander@4: 
dvander@21: static inline bool
dvander@21: IsHexDigit(char c)
dvander@21: {
dvander@21:     return IsDigit(c) ||
dvander@21:            (c >= 'a' && c <= 'f') ||
dvander@21:            (c >= 'A' && c <= 'F');
dvander@21: }
dvander@21: 
dvander@21: // Based on the logic for litchar() in sc2.c.
dvander@21: int
dvander@21: Scanner::readEscapeCode()
dvander@21: {
dvander@21:     char c = getChar();
dvander@21:     if (c == '\\')
dvander@21:         return c;
dvander@21: 
dvander@21:     switch (c) {
dvander@21:       case 'a':
dvander@21:         return '\a';
dvander@21:       case 'b':
dvander@21:         return '\b';
dvander@21:       case 'e':
dvander@21:         return 27;          // Apparently \e is non-standard.
dvander@21:       case 'f':
dvander@21:         return '\f';
dvander@21:       case 'n':
dvander@21:         return '\n';
dvander@21:       case 'r':
dvander@21:         return '\r';
dvander@21:       case 't':
dvander@21:         return '\t';
dvander@21:       case '\v':
dvander@21:         return '\v';
dvander@21: 
dvander@21:       case 'x':
dvander@21:       {
dvander@21:           unsigned digits = 0;
dvander@21:           char r = 0;
dvander@21:   
dvander@21:           c = getChar();
dvander@21:           while (IsHexDigit(c) && digits < 2) {
dvander@21:               if (IsDigit(c))
dvander@21:                   c = (c << 4) + (c - '0');
dvander@21:               else
dvander@21:                   c = (c << 4) + (tolower(c) - 'a' + 10);
dvander@21:               digits++;
dvander@21:               c = getChar();
dvander@21:            }
dvander@21:                 
dvander@21:           // Swallow a trailing ';'
dvander@21:           if (c != ';')
dvander@21:               putChar(c);
dvander@21:   
dvander@21:           return r;
dvander@21:       }
dvander@21: 
dvander@21:       case '\'':
dvander@21:       case '\"':
dvander@21:       case '%':
dvander@21:         return c;
dvander@21: 
dvander@21:       default:
dvander@21:       {
dvander@21:         if (IsDigit(c)) {
dvander@21:             // \ddd
dvander@21:             char r = 0;
dvander@21:             while (IsDigit(c)) {
dvander@21:                 r = r * 10 + (c - '0');
dvander@21:                 c = getChar();
dvander@21:             }
dvander@21: 
dvander@21:             // Swallow a trailing ;
dvander@21:             if (c != ';')
dvander@21:                 putChar(c);
dvander@21: 
dvander@21:             return r;
dvander@21:         }
dvander@21:         assert(false);
dvander@21:         return INT_MAX;
dvander@21:       }
dvander@21:     }
dvander@21: }
dvander@21: 
dvander@21: TokenKind
dvander@21: Scanner::stringLiteral()
dvander@21: {
dvander@21:     literal_.clear();
dvander@21: 
dvander@21:     for (;;) {
dvander@21:         char c = getChar();
dvander@21:         if (c == '\"')
dvander@21:             break;
dvander@21:         if (c == '\r' || c == '\n' || c == '\0') {
dvander@21:             assert(false);
dvander@21:             return TOK_ERROR;
dvander@21:         }
dvander@21:         if (c == '\\') {
dvander@21:             int code = readEscapeCode();
dvander@21:             if (code == INT_MAX)
dvander@21:                 return TOK_ERROR;
dvander@21:             c = char(code);
dvander@21:         }
dvander@21:         literal_ += c;
dvander@21:     }
dvander@21: 
dvander@21:     return TOK_STRING_LITERAL;
dvander@21: }
dvander@21: 
dvander@0: TokenKind
dvander@4: Scanner::numberLiteral(char first)
dvander@0: {
dvander@0:     literal_ = first;
dvander@0: 
dvander@4:     char c;
dvander@0:     for (;;) {
dvander@4:         c = getChar();
dvander@4:         if (!IsDigit(c))
dvander@0:             break;
dvander@0:         literal_ += c;
dvander@0:     }
dvander@0: 
dvander@4:     if (c != '.') {
dvander@4:         putChar(c);
dvander@4:         return TOK_INTEGER_LITERAL;
dvander@4:     }
dvander@4:     literal_ += c;
dvander@4: 
dvander@4:     c = getChar();
dvander@4:     if (!IsDigit(c)) {
dvander@4:         assert(false);
dvander@4:         return TOK_ERROR;
dvander@4:     }
dvander@4:     literal_ += c;
dvander@4: 
dvander@4:     for (;;) {
dvander@4:         c = getChar();
dvander@4:         if (!IsDigit(c))
dvander@4:             break;
dvander@4:         literal_ += c;
dvander@4:     }
dvander@4: 
dvander@4:     if (c != 'e') {
dvander@4:         putChar(c);
dvander@4:     } else {
dvander@4:         literal_ += c;
dvander@4:         c = getChar();
dvander@4:         if (c == '-') {
dvander@4:             literal_ += c;
dvander@4:             c = getChar();
dvander@4:         }
dvander@4:         if (!IsDigit(c)) {
dvander@4:             assert(false);
dvander@4:             return TOK_ERROR;
dvander@4:         }
dvander@4:         literal_ += c;
dvander@4:         for (;;) {
dvander@4:             c = getChar();
dvander@4:             if (!IsDigit(c)) {
dvander@4:                 putChar(c);
dvander@4:                 break;
dvander@4:             }
dvander@4:             literal_ += c;
dvander@4:         }
dvander@4:     }
dvander@4: 
dvander@4:     return TOK_FLOAT_LITERAL;
dvander@0: }
dvander@0: 
dvander@0: TokenKind
dvander@0: Scanner::identifier(char first)
dvander@0: {
dvander@0:     literal_ = first;
dvander@0: 
dvander@4:     char c;
dvander@0:     for (;;) {
dvander@4:         c = getChar();
dvander@0:         if (!isascii(c) || (!isalnum(c) && c != '_')) {
dvander@4:             if (c != ':')
dvander@4:                 putChar(c);
dvander@0:             break;
dvander@0:         }
dvander@0:         literal_ += c;
dvander@0:     }
dvander@0: 
dvander@0:     if (literal_ == "new")
dvander@0:         return TOK_NEW;
dvander@0:     if (literal_ == "for")
dvander@0:         return TOK_FOR;
dvander@0:     if (literal_ == "return")
dvander@0:         return TOK_RETURN;
dvander@0:     if (literal_ == "if")
dvander@0:         return TOK_IF;
dvander@0:     if (literal_ == "else")
dvander@0:         return TOK_ELSE;
dvander@2:     if (literal_ == "enum")
dvander@2:         return TOK_ENUM;
dvander@9:     if (literal_ == "while")
dvander@9:         return TOK_WHILE;
dvander@9:     if (literal_ == "do")
dvander@9:         return TOK_DO;
dvander@10:     if (literal_ == "break")
dvander@10:         return TOK_BREAK;
dvander@10:     if (literal_ == "continue")
dvander@10:         return TOK_CONTINUE;
dvander@4:     return (c == ':') ? TOK_LABEL : TOK_NAME;
dvander@0: }
dvander@0: 
dvander@0: TokenKind
dvander@0: Scanner::next()
dvander@0: {
dvander@0:     assert(!pushed_);
dvander@0: 
dvander@0:     for (;;) {
dvander@0:         char c = getChar();
dvander@0:         switch (c) {
dvander@0:           case '/':
dvander@0:           {
dvander@0:             c = getChar();
dvander@0:             if (c == '/') {
dvander@0:                 eatSingleLine();
dvander@0:                 break;
dvander@0:             }
dvander@0:             if (c == '*') {
dvander@0:                 if (!eatComment())
dvander@0:                     return TOK_ERROR;
dvander@0:                 break;
dvander@0:             }
dvander@0:             putChar(c);
dvander@0:             return TOK_SLASH;
dvander@0:           }
dvander@0: 
dvander@0:           case ';': return TOK_SEMICOLON;
dvander@0:           case '{': return TOK_LBRACE;
dvander@0:           case '}': return TOK_RBRACE;
dvander@0:           case '(': return TOK_LPAREN;
dvander@0:           case ')': return TOK_RPAREN;
dvander@0:           case '[': return TOK_LBRACKET;
dvander@0:           case ']': return TOK_RBRACKET;
dvander@2: 
dvander@2:           case '*':
dvander@2:             c = getChar();
dvander@2:             if (c == '=')
dvander@2:                 return TOK_ASSIGN_MUL;
dvander@2:             putChar(c);
dvander@2:             return TOK_STAR;
dvander@2: 
dvander@2:           case '+':
dvander@2:             c = getChar();
dvander@2:             if (c == '=')
dvander@2:                 return TOK_ASSIGN_ADD;
dvander@23:             if (c == '+')
dvander@23:                 return TOK_INCREMENT;
dvander@2:             putChar(c);
dvander@2:             return TOK_PLUS;
dvander@2: 
dvander@8:           case '&':
dvander@22:             c = getChar();
dvander@22:             if (c == '=')
dvander@22:                 return TOK_ASSIGN_BITAND;
dvander@22:             putChar(c);
dvander@8:             return TOK_BITAND;
dvander@8: 
dvander@8:           case '|':
dvander@22:             c = getChar();
dvander@22:             if (c == '=')
dvander@22:                 return TOK_ASSIGN_BITOR;
dvander@22:             putChar(c);
dvander@8:             return TOK_BITOR;
dvander@8: 
dvander@8:           case '^':
dvander@22:             c = getChar();
dvander@22:             if (c == '=')
dvander@22:                 return TOK_ASSIGN_BITXOR;
dvander@22:             putChar(c);
dvander@8:             return TOK_BITXOR;
dvander@8: 
dvander@22:           case '-':
dvander@22:             c = getChar();
dvander@22:             if (c == '=')
dvander@22:                 return TOK_ASSIGN_SUB;
dvander@23:             if (c == '-')
dvander@23:                 return TOK_DECREMENT;
dvander@22:             putChar(c);
dvander@22:             return TOK_MINUS;
dvander@22: 
dvander@0:           case ',': return TOK_COMMA;
dvander@0:           case '\0': return TOK_EOF;
dvander@0: 
dvander@0:           case '=':
dvander@0:             c = getChar();
dvander@0:             if (c == '=')
dvander@0:                 return TOK_EQUALS;
dvander@0:             putChar(c);
dvander@0:             return TOK_ASSIGN;
dvander@0: 
dvander@0:           case '<':
dvander@0:             c = getChar();
dvander@0:             if (c == '=')
dvander@0:                 return TOK_LE;
dvander@2:             if (c == '<') {
dvander@2:                 c = getChar();
dvander@2:                 if (c == '=')
dvander@2:                     return TOK_ASSIGN_SHL;
dvander@2:                 return TOK_SHL;
dvander@2:             }
dvander@0:             putChar(c);
dvander@0:             return TOK_LT;
dvander@0: 
dvander@0:           case '>':
dvander@0:             c = getChar();
dvander@0:             if (c == '=')
dvander@0:                 return TOK_GE;
dvander@8:             if (c == '>') {
dvander@8:                 c = getChar();
dvander@22:                 if (c == '>') {
dvander@22:                     c = getChar();
dvander@22:                     if (c == '=')
dvander@22:                         return TOK_ASSIGN_USHR;
dvander@22:                     putChar(c);
dvander@8:                     return TOK_USHR;
dvander@22:                 }
dvander@8:                 return TOK_SHR;
dvander@8:             }
dvander@0:             putChar(c);
dvander@0:             return TOK_GT;
dvander@0: 
dvander@0:           case ' ':
dvander@0:           case '\t':
dvander@0:           case '\r':
dvander@0:           case '\f':
dvander@0:           case '\n':
dvander@0:             break;
dvander@0: 
dvander@21:           case '"':
dvander@21:             return stringLiteral();
dvander@21: 
dvander@0:           default:
dvander@0:             if (c >= '0' && c <= '9')
dvander@4:                 return numberLiteral(c);
dvander@0:             if (c == '_' || (isascii(c) && isalpha(c)))
dvander@0:                 return identifier(c);
dvander@4:             assert(false);
dvander@0:             return TOK_ERROR;
dvander@0:         }
dvander@0:     }
dvander@0: }
dvander@0: 
dvander@0: TokenKind
dvander@0: Scanner::scan()
dvander@0: {
dvander@0:     if (pushed_) {
dvander@0:         pushed_ = false;
dvander@0:         return lastToken_;
dvander@0:     }
dvander@0:     lastToken_ = next();
dvander@0:     return lastToken_;
dvander@0: }
dvander@0: 
dvander@0: TokenKind
dvander@0: Scanner::peek()
dvander@0: {
dvander@0:     if (pushed_)
dvander@0:         return lastToken_;
dvander@0:     scan();
dvander@0:     pushed_ = true;
dvander@0:     return lastToken_;
dvander@0: }