jitcraft/Scanner.cpp
author David Anderson <dvander@alliedmods.net>
Sat Jun 30 22:45:25 2012 -0700 (2012-06-30)
changeset 16 23627f8bc421
parent 10 0c3bcd50e361
child 21 af5d8e985502
permissions -rw-r--r--
Added support for arbitrary expression references.
dvander@0
     1
/*
dvander@0
     2
 * Copyright (C) 2012 David Anderson
dvander@0
     3
 *
dvander@0
     4
 * This file is part of JITCraft.
dvander@0
     5
 *
dvander@0
     6
 * JITCraft is free software: you can redistribute it and/or modify it under
dvander@0
     7
 * the terms of the GNU General Public License as published by the Free
dvander@0
     8
 * Software Foundation, either version 3 of the License, or (at your option)
dvander@0
     9
 * any later version.
dvander@0
    10
 * 
dvander@0
    11
 * Foobar is distributed in the hope that it will be useful, but WITHOUT ANY
dvander@0
    12
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
dvander@0
    13
 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
dvander@0
    14
 *
dvander@0
    15
 * You should have received a copy of the GNU General Public License along with
dvander@0
    16
 * JITCraft. If not, see http://www.gnu.org/licenses/.
dvander@0
    17
 */
dvander@0
    18
#include <assert.h>
dvander@0
    19
#include <ctype.h>
dvander@0
    20
#include "Scanner.h"
dvander@0
    21
dvander@0
    22
using namespace jc;
dvander@0
    23
dvander@0
    24
const char *jc::TokenNames[] =
dvander@0
    25
{
dvander@0
    26
#define _(name, str) str,
dvander@0
    27
    TOKENMAP(_)
dvander@0
    28
#undef _
dvander@0
    29
    NULL
dvander@0
    30
};
dvander@0
    31
dvander@0
    32
Scanner::Scanner(const char *stream, size_t length)
dvander@0
    33
  : stream_(stream),
dvander@0
    34
    length_(length),
dvander@0
    35
    pushed_(false),
dvander@0
    36
    saved_('\0'),
dvander@0
    37
    pos_(stream_)
dvander@0
    38
{
dvander@0
    39
}
dvander@0
    40
dvander@0
    41
char
dvander@0
    42
Scanner::getChar()
dvander@0
    43
{
dvander@0
    44
    if (saved_ != '\0') {
dvander@0
    45
        char temp = saved_;
dvander@0
    46
        saved_ = '\0';
dvander@0
    47
        return temp;
dvander@0
    48
    }
dvander@0
    49
    if (pos_ >= stream_ + length_)
dvander@0
    50
        return '\0';
dvander@0
    51
    return *pos_++;
dvander@0
    52
}
dvander@0
    53
dvander@0
    54
void
dvander@0
    55
Scanner::putChar(char c)
dvander@0
    56
{
dvander@0
    57
    assert(saved_ == '\0');
dvander@0
    58
    saved_ = c;
dvander@0
    59
}
dvander@0
    60
dvander@0
    61
void
dvander@0
    62
Scanner::eatSingleLine()
dvander@0
    63
{
dvander@0
    64
    char c;
dvander@0
    65
    while ((c = getChar()) != '\0') {
dvander@0
    66
        if (c == '\n' || c == '\r')
dvander@0
    67
            break;
dvander@0
    68
    }
dvander@0
    69
}
dvander@0
    70
dvander@0
    71
bool
dvander@0
    72
Scanner::eatComment()
dvander@0
    73
{
dvander@0
    74
    char c;
dvander@0
    75
dvander@0
    76
    while ((c = getChar()) != '\0') {
dvander@0
    77
        while (c == '*') {
dvander@0
    78
            c = getChar();
dvander@0
    79
            if (c == '\0')
dvander@0
    80
                return false;
dvander@0
    81
            if (c == '/')
dvander@0
    82
                return true;
dvander@0
    83
        }
dvander@0
    84
    }
dvander@0
    85
dvander@0
    86
    return false;
dvander@0
    87
}
dvander@0
    88
dvander@4
    89
static inline bool
dvander@4
    90
IsDigit(char c)
dvander@4
    91
{
dvander@4
    92
    return c >= '0' && c <= '9';
dvander@4
    93
}
dvander@4
    94
dvander@4
    95
// Based off the logic in sc2.c's ftoi()...
dvander@4
    96
double
dvander@4
    97
Scanner::ParseDouble(const char *string)
dvander@4
    98
{
dvander@4
    99
    const char *ptr = string;
dvander@4
   100
dvander@4
   101
    double number = 0.0;
dvander@4
   102
    while (IsDigit(*ptr)) {
dvander@4
   103
        number = (number * 10) + (*ptr - '0');
dvander@4
   104
        ptr++;
dvander@4
   105
    }
dvander@4
   106
dvander@4
   107
    assert(*ptr == '.');
dvander@4
   108
    ptr++;
dvander@4
   109
dvander@4
   110
    double fraction = 0.0;
dvander@4
   111
    double multiplier = 1.0;
dvander@4
   112
    while (IsDigit(*ptr)) {
dvander@4
   113
        fraction = (fraction * 10) + (*ptr - '0');
dvander@4
   114
        multiplier = multiplier / 10.0;
dvander@4
   115
        ptr++;
dvander@4
   116
    }
dvander@4
   117
dvander@4
   118
    number += fraction * multiplier;
dvander@4
   119
dvander@4
   120
    if (*ptr++ == 'e') {
dvander@4
   121
        int sign = 1;
dvander@4
   122
        if (*ptr == '-') {
dvander@4
   123
            sign = -1;
dvander@4
   124
            ptr++;
dvander@4
   125
        }
dvander@4
   126
dvander@4
   127
        int exponent = 0;
dvander@4
   128
        while (IsDigit(*ptr)) {
dvander@4
   129
            exponent = (exponent * 10) + (*ptr - '0');
dvander@4
   130
            ptr++;
dvander@4
   131
        }
dvander@4
   132
dvander@4
   133
        multiplier = pow(10.0, exponent * sign);
dvander@4
   134
        number *= multiplier;
dvander@4
   135
    }
dvander@4
   136
dvander@4
   137
    return number;
dvander@4
   138
}
dvander@4
   139
dvander@0
   140
TokenKind
dvander@4
   141
Scanner::numberLiteral(char first)
dvander@0
   142
{
dvander@0
   143
    literal_ = first;
dvander@0
   144
dvander@4
   145
    char c;
dvander@0
   146
    for (;;) {
dvander@4
   147
        c = getChar();
dvander@4
   148
        if (!IsDigit(c))
dvander@0
   149
            break;
dvander@0
   150
        literal_ += c;
dvander@0
   151
    }
dvander@0
   152
dvander@4
   153
    if (c != '.') {
dvander@4
   154
        putChar(c);
dvander@4
   155
        return TOK_INTEGER_LITERAL;
dvander@4
   156
    }
dvander@4
   157
    literal_ += c;
dvander@4
   158
dvander@4
   159
    c = getChar();
dvander@4
   160
    if (!IsDigit(c)) {
dvander@4
   161
        assert(false);
dvander@4
   162
        return TOK_ERROR;
dvander@4
   163
    }
dvander@4
   164
    literal_ += c;
dvander@4
   165
dvander@4
   166
    for (;;) {
dvander@4
   167
        c = getChar();
dvander@4
   168
        if (!IsDigit(c))
dvander@4
   169
            break;
dvander@4
   170
        literal_ += c;
dvander@4
   171
    }
dvander@4
   172
dvander@4
   173
    if (c != 'e') {
dvander@4
   174
        putChar(c);
dvander@4
   175
    } else {
dvander@4
   176
        literal_ += c;
dvander@4
   177
        c = getChar();
dvander@4
   178
        if (c == '-') {
dvander@4
   179
            literal_ += c;
dvander@4
   180
            c = getChar();
dvander@4
   181
        }
dvander@4
   182
        if (!IsDigit(c)) {
dvander@4
   183
            assert(false);
dvander@4
   184
            return TOK_ERROR;
dvander@4
   185
        }
dvander@4
   186
        literal_ += c;
dvander@4
   187
        for (;;) {
dvander@4
   188
            c = getChar();
dvander@4
   189
            if (!IsDigit(c)) {
dvander@4
   190
                putChar(c);
dvander@4
   191
                break;
dvander@4
   192
            }
dvander@4
   193
            literal_ += c;
dvander@4
   194
        }
dvander@4
   195
    }
dvander@4
   196
dvander@4
   197
    return TOK_FLOAT_LITERAL;
dvander@0
   198
}
dvander@0
   199
dvander@0
   200
TokenKind
dvander@0
   201
Scanner::identifier(char first)
dvander@0
   202
{
dvander@0
   203
    literal_ = first;
dvander@0
   204
dvander@4
   205
    char c;
dvander@0
   206
    for (;;) {
dvander@4
   207
        c = getChar();
dvander@0
   208
        if (!isascii(c) || (!isalnum(c) && c != '_')) {
dvander@4
   209
            if (c != ':')
dvander@4
   210
                putChar(c);
dvander@0
   211
            break;
dvander@0
   212
        }
dvander@0
   213
        literal_ += c;
dvander@0
   214
    }
dvander@0
   215
dvander@0
   216
    if (literal_ == "new")
dvander@0
   217
        return TOK_NEW;
dvander@0
   218
    if (literal_ == "for")
dvander@0
   219
        return TOK_FOR;
dvander@0
   220
    if (literal_ == "return")
dvander@0
   221
        return TOK_RETURN;
dvander@0
   222
    if (literal_ == "if")
dvander@0
   223
        return TOK_IF;
dvander@0
   224
    if (literal_ == "else")
dvander@0
   225
        return TOK_ELSE;
dvander@2
   226
    if (literal_ == "enum")
dvander@2
   227
        return TOK_ENUM;
dvander@9
   228
    if (literal_ == "while")
dvander@9
   229
        return TOK_WHILE;
dvander@9
   230
    if (literal_ == "do")
dvander@9
   231
        return TOK_DO;
dvander@10
   232
    if (literal_ == "break")
dvander@10
   233
        return TOK_BREAK;
dvander@10
   234
    if (literal_ == "continue")
dvander@10
   235
        return TOK_CONTINUE;
dvander@4
   236
    return (c == ':') ? TOK_LABEL : TOK_NAME;
dvander@0
   237
}
dvander@0
   238
dvander@0
   239
TokenKind
dvander@0
   240
Scanner::next()
dvander@0
   241
{
dvander@0
   242
    assert(!pushed_);
dvander@0
   243
dvander@0
   244
    for (;;) {
dvander@0
   245
        char c = getChar();
dvander@0
   246
        switch (c) {
dvander@0
   247
          case '/':
dvander@0
   248
          {
dvander@0
   249
            c = getChar();
dvander@0
   250
            if (c == '/') {
dvander@0
   251
                eatSingleLine();
dvander@0
   252
                break;
dvander@0
   253
            }
dvander@0
   254
            if (c == '*') {
dvander@0
   255
                if (!eatComment())
dvander@0
   256
                    return TOK_ERROR;
dvander@0
   257
                break;
dvander@0
   258
            }
dvander@0
   259
            putChar(c);
dvander@0
   260
            return TOK_SLASH;
dvander@0
   261
          }
dvander@0
   262
dvander@0
   263
          case ';': return TOK_SEMICOLON;
dvander@0
   264
          case '{': return TOK_LBRACE;
dvander@0
   265
          case '}': return TOK_RBRACE;
dvander@0
   266
          case '(': return TOK_LPAREN;
dvander@0
   267
          case ')': return TOK_RPAREN;
dvander@0
   268
          case '[': return TOK_LBRACKET;
dvander@0
   269
          case ']': return TOK_RBRACKET;
dvander@2
   270
dvander@2
   271
          case '*':
dvander@2
   272
            c = getChar();
dvander@2
   273
            if (c == '=')
dvander@2
   274
                return TOK_ASSIGN_MUL;
dvander@2
   275
            putChar(c);
dvander@2
   276
            return TOK_STAR;
dvander@2
   277
dvander@2
   278
          case '+':
dvander@2
   279
            c = getChar();
dvander@2
   280
            if (c == '=')
dvander@2
   281
                return TOK_ASSIGN_ADD;
dvander@2
   282
            putChar(c);
dvander@2
   283
            return TOK_PLUS;
dvander@2
   284
dvander@8
   285
          case '&':
dvander@8
   286
            return TOK_BITAND;
dvander@8
   287
dvander@8
   288
          case '|':
dvander@8
   289
            return TOK_BITOR;
dvander@8
   290
dvander@8
   291
          case '^':
dvander@8
   292
            return TOK_BITXOR;
dvander@8
   293
dvander@0
   294
          case '-': return TOK_MINUS;
dvander@0
   295
          case ',': return TOK_COMMA;
dvander@0
   296
          case '\0': return TOK_EOF;
dvander@0
   297
dvander@0
   298
          case '=':
dvander@0
   299
            c = getChar();
dvander@0
   300
            if (c == '=')
dvander@0
   301
                return TOK_EQUALS;
dvander@0
   302
            putChar(c);
dvander@0
   303
            return TOK_ASSIGN;
dvander@0
   304
dvander@0
   305
          case '<':
dvander@0
   306
            c = getChar();
dvander@0
   307
            if (c == '=')
dvander@0
   308
                return TOK_LE;
dvander@2
   309
            if (c == '<') {
dvander@2
   310
                c = getChar();
dvander@2
   311
                if (c == '=')
dvander@2
   312
                    return TOK_ASSIGN_SHL;
dvander@2
   313
                return TOK_SHL;
dvander@2
   314
            }
dvander@0
   315
            putChar(c);
dvander@0
   316
            return TOK_LT;
dvander@0
   317
dvander@0
   318
          case '>':
dvander@0
   319
            c = getChar();
dvander@0
   320
            if (c == '=')
dvander@0
   321
                return TOK_GE;
dvander@8
   322
            if (c == '>') {
dvander@8
   323
                c = getChar();
dvander@8
   324
                if (c == '>')
dvander@8
   325
                    return TOK_USHR;
dvander@8
   326
                return TOK_SHR;
dvander@8
   327
            }
dvander@0
   328
            putChar(c);
dvander@0
   329
            return TOK_GT;
dvander@0
   330
dvander@0
   331
          case ' ':
dvander@0
   332
          case '\t':
dvander@0
   333
          case '\r':
dvander@0
   334
          case '\f':
dvander@0
   335
          case '\n':
dvander@0
   336
            break;
dvander@0
   337
dvander@0
   338
          default:
dvander@0
   339
            if (c >= '0' && c <= '9')
dvander@4
   340
                return numberLiteral(c);
dvander@0
   341
            if (c == '_' || (isascii(c) && isalpha(c)))
dvander@0
   342
                return identifier(c);
dvander@4
   343
            assert(false);
dvander@0
   344
            return TOK_ERROR;
dvander@0
   345
        }
dvander@0
   346
    }
dvander@0
   347
}
dvander@0
   348
dvander@0
   349
TokenKind
dvander@0
   350
Scanner::scan()
dvander@0
   351
{
dvander@0
   352
    if (pushed_) {
dvander@0
   353
        pushed_ = false;
dvander@0
   354
        return lastToken_;
dvander@0
   355
    }
dvander@0
   356
    lastToken_ = next();
dvander@0
   357
    return lastToken_;
dvander@0
   358
}
dvander@0
   359
dvander@0
   360
TokenKind
dvander@0
   361
Scanner::peek()
dvander@0
   362
{
dvander@0
   363
    if (pushed_)
dvander@0
   364
        return lastToken_;
dvander@0
   365
    scan();
dvander@0
   366
    pushed_ = true;
dvander@0
   367
    return lastToken_;
dvander@0
   368
}