tools/incparser/IncParser.cs
author Scott Ehlert <ds@alliedmods.net>
Sun Aug 24 05:05:04 2008 +0000 (2008-08-24)
changeset 2238 0879e1742145
parent 2019 9f473114a511
child 2765 ace14b1768dc
permissions -rw-r--r--
Fixed build.

"Nice" improvement DS.
ds@1952
     1
using System;
ds@1952
     2
using System.Collections.Generic;
ds@1952
     3
using System.IO;
ds@1952
     4
using System.Text;
ds@1952
     5
ds@1952
     6
namespace incparser
ds@1952
     7
{
ds@1952
     8
    enum LexToken
ds@1952
     9
    {
ds@1952
    10
        TOKEN_NONE,
ds@1952
    11
        TOKEN_INCLUDE,
ds@1952
    12
        TOKEN_DEFINE,
ds@1952
    13
        TOKEN_CONST,
ds@1952
    14
        TOKEN_STRUCT,
ds@1952
    15
        TOKEN_FUNCTAG,
ds@1952
    16
        TOKEN_FUNCENUM,
ds@1952
    17
        TOKEN_ENUM,
ds@1952
    18
        TOKEN_NATIVE,
ds@1952
    19
        TOKEN_STOCK,
ds@1952
    20
        TOKEN_PUBLIC,
ds@1952
    21
        TOKEN_FORWARD,
ds@1952
    22
        TOKEN_QUOTCHAR,
ds@1952
    23
        TOKEN_OPERATOR,
ds@1952
    24
        TOKEN_CHARACTER,
ds@1952
    25
        /*TOKEN_STRING,*/
ds@1952
    26
        TOKEN_LABEL,
ds@1952
    27
        TOKEN_SYMBOL,
ds@1952
    28
        TOKEN_DOCBLOCK,
ds@1952
    29
        TOKEN_EOF,
ds@1952
    30
    }
ds@1952
    31
ds@1952
    32
    class ParseException : System.Exception
ds@1952
    33
    {
ds@1952
    34
        public ParseException(string message) : base(message)
ds@1952
    35
        {
ds@1952
    36
        }
ds@1952
    37
    };
ds@1952
    38
         
ds@1952
    39
    class KeywordToken
ds@1952
    40
    {
ds@1952
    41
        public KeywordToken(string k, LexToken t)
ds@1952
    42
        {
ds@1952
    43
            keyword = k;
ds@1952
    44
            token = t;
ds@1952
    45
        }
ds@1952
    46
        public string keyword;
ds@1952
    47
        public LexToken token;
ds@1952
    48
    };
ds@1952
    49
ds@1952
    50
    class Tokenizer
ds@1952
    51
    {
ds@1952
    52
        private static bool s_initialized = false;
ds@1952
    53
        private static int s_position = 0;
ds@1952
    54
        private static KeywordToken [] s_tokens = null;
ds@1952
    55
        public static KeywordToken[] Tokens = null;
ds@1952
    56
ds@1952
    57
        public static void Initialize()
ds@1952
    58
        {
ds@1952
    59
            if (s_initialized)
ds@1952
    60
            {
ds@1952
    61
                return;
ds@1952
    62
            }
ds@1952
    63
            s_tokens = new KeywordToken[12];
ds@1952
    64
            s_initialized = true;
ds@1952
    65
            s_position = 0;
ds@1952
    66
ds@1952
    67
            AddToken("#include", LexToken.TOKEN_INCLUDE);
ds@1952
    68
            AddToken("#define", LexToken.TOKEN_DEFINE);
ds@1952
    69
            AddToken("native", LexToken.TOKEN_NATIVE);
ds@1952
    70
            AddToken("functag", LexToken.TOKEN_FUNCTAG);
ds@1952
    71
            AddToken("funcenum", LexToken.TOKEN_FUNCENUM);
ds@1952
    72
            AddToken("enum", LexToken.TOKEN_ENUM);
ds@1952
    73
            AddToken("stock", LexToken.TOKEN_STOCK);
ds@1952
    74
            AddToken("public", LexToken.TOKEN_PUBLIC);
ds@1952
    75
            AddToken("forward", LexToken.TOKEN_FORWARD);
ds@1952
    76
            AddToken("const", LexToken.TOKEN_CONST);
ds@1952
    77
            AddToken("struct", LexToken.TOKEN_STRUCT);
ds@1952
    78
            AddToken("operator", LexToken.TOKEN_OPERATOR);
ds@1952
    79
ds@1952
    80
            Tokens = s_tokens;
ds@1952
    81
        }
ds@1952
    82
ds@1952
    83
        private static void AddToken(string lex, LexToken tok)
ds@1952
    84
        {
ds@1952
    85
            s_tokens[s_position] = new KeywordToken(lex, tok);
ds@1952
    86
            s_position++;
ds@1952
    87
        }
ds@1952
    88
    };
ds@1952
    89
ds@1952
    90
    class IncParser
ds@1952
    91
    {
ds@1952
    92
        private string FileName;
ds@1952
    93
        private StringBuilder Contents;
ds@1952
    94
        private bool LexPushed;
ds@1952
    95
        private LexToken _LastToken;
ds@1952
    96
        private string _LexString;
ds@1952
    97
        private char _LexChar;
ds@1952
    98
        private uint LineNo;
ds@1952
    99
ds@1952
   100
        public string LexString
ds@1952
   101
        {
ds@1952
   102
            get
ds@1952
   103
            {
ds@1952
   104
                return _LexString;
ds@1952
   105
            }
ds@1952
   106
        }
ds@1952
   107
ds@1952
   108
        public char LexChar
ds@1952
   109
        {
ds@1952
   110
            get
ds@1952
   111
            {
ds@1952
   112
                return _LexChar;
ds@1952
   113
            }
ds@1952
   114
        }
ds@1952
   115
ds@1952
   116
        public uint GetLineNumber()
ds@1952
   117
        {
ds@1952
   118
            return LineNo;
ds@1952
   119
        }
ds@1952
   120
ds@1952
   121
        public IncParser(string file)
ds@1952
   122
        {
ds@1952
   123
            FileName = file;
ds@1952
   124
ds@1952
   125
            /* Clear out lexer stuff */
ds@1952
   126
            LexPushed = false;
ds@1952
   127
            _LastToken = LexToken.TOKEN_NONE;
ds@1952
   128
            _LexString = null;
ds@1952
   129
            _LexChar = '\0';
ds@1952
   130
            LineNo = 1;
ds@1952
   131
ds@1952
   132
            /* Initialize; this can throw an exception */
ds@1952
   133
            Initialize();
ds@1952
   134
        }
ds@1952
   135
pred@2019
   136
         public void Parse(ParseWriter w)
ds@1952
   137
        {
ds@1952
   138
            w.BeginSection(System.IO.Path.GetFileName(FileName));
ds@1952
   139
            LexToken tok = LexToken.TOKEN_NONE;
ds@1952
   140
            while ((tok = lex()) != LexToken.TOKEN_EOF)
ds@1952
   141
            {
ds@1952
   142
                switch (tok)
ds@1952
   143
                {
ds@1952
   144
                    case LexToken.TOKEN_DOCBLOCK:
ds@1952
   145
                        {
ds@1952
   146
                            w.WritePair("doc", LexString);
ds@1952
   147
                            break;
ds@1952
   148
                        }
ds@1952
   149
                    case LexToken.TOKEN_DEFINE:
ds@1952
   150
                        {
ds@1952
   151
                            PARSE_Define(w);
ds@1952
   152
                            break;
ds@1952
   153
                        }
ds@1952
   154
                    case LexToken.TOKEN_ENUM:
ds@1952
   155
                        {
ds@1952
   156
                            PARSE_Enum(w);
ds@1952
   157
                            break;
ds@1952
   158
                        }
ds@1952
   159
                    case LexToken.TOKEN_FORWARD:
ds@1952
   160
                        {
ds@1952
   161
                            PARSE_Function(tok, w);
ds@1952
   162
                            break;
ds@1952
   163
                        }
ds@1952
   164
                    case LexToken.TOKEN_NATIVE:
ds@1952
   165
                        {
ds@1952
   166
                            PARSE_Function(tok, w);
ds@1952
   167
                            break;
ds@1952
   168
                        }
ds@1952
   169
                    case LexToken.TOKEN_STOCK:
ds@1952
   170
                        {
ds@1952
   171
                            PARSE_Function(tok, w);
ds@1952
   172
                            break;
ds@1952
   173
                        }
ds@1952
   174
                    case LexToken.TOKEN_FUNCTAG:
ds@1952
   175
                        {
ds@1952
   176
                            PARSE_FuncTag(w);
ds@1952
   177
                            break;
ds@1952
   178
                        }
ds@1952
   179
                    case LexToken.TOKEN_FUNCENUM:
ds@1952
   180
                        {
ds@1952
   181
                            PARSE_FuncEnum(w);
ds@1952
   182
                            break;
ds@1952
   183
                        }
ds@1952
   184
                    case LexToken.TOKEN_INCLUDE:
ds@1952
   185
                        {
ds@1952
   186
                            PARSE_Include(w);
ds@1952
   187
                            break;
ds@1952
   188
                        }
ds@1952
   189
                    case LexToken.TOKEN_STRUCT:
ds@1952
   190
                        {
ds@1952
   191
                            PARSE_Struct(w);
ds@1952
   192
                            break;
ds@1952
   193
                        }
ds@1952
   194
                    case LexToken.TOKEN_PUBLIC:
ds@1952
   195
                        {
ds@1952
   196
                            PARSE_Public(w);
ds@1952
   197
                            break;
ds@1952
   198
                        }
ds@1952
   199
                    default:
ds@1952
   200
                        {
pred@2019
   201
                            if (_LexString == "static" || _LexString == "new")
pred@2019
   202
                            {
pred@2019
   203
                                PARSE_Public(w);
pred@2019
   204
                                break;
pred@2019
   205
                            }
pred@2019
   206
                            throw new ParseException("Unrecognized token: " + tok + " " + _LexString);
ds@1952
   207
                        }
ds@1952
   208
                }
ds@1952
   209
            }
ds@1952
   210
            w.EndSection();
ds@1952
   211
        }
ds@1952
   212
ds@1952
   213
        private void PARSE_Public(ParseWriter w)
ds@1952
   214
        {
pred@2019
   215
            MatchToken(LexToken.TOKEN_CONST);
pred@2019
   216
ds@1952
   217
            /* Eat up an optional tag */
ds@1952
   218
            MatchToken(LexToken.TOKEN_LABEL);
pred@2019
   219
ds@1952
   220
            /* Eat up a name */
ds@1952
   221
            NeedToken(LexToken.TOKEN_SYMBOL);
ds@1952
   222
pred@2019
   223
            while (MatchChar('['))
pred@2019
   224
            {
pred@2019
   225
                DiscardUntilChar(']');
pred@2019
   226
                NeedChar(']');
pred@2019
   227
            }
pred@2019
   228
ds@1952
   229
            if (MatchChar('='))
ds@1952
   230
            {
ds@1952
   231
                ignore_block();
ds@1952
   232
            }
pred@2019
   233
            else if (MatchChar('(') && MatchChar(')'))
pred@2019
   234
            {
pred@2019
   235
                ignore_block();
pred@2019
   236
                return;
pred@2019
   237
            }
ds@1952
   238
ds@1952
   239
            NeedChar(';');
ds@1952
   240
        }
ds@1952
   241
ds@1952
   242
        private void PARSE_Struct(ParseWriter w)
ds@1952
   243
        {
ds@1952
   244
            /* For now, we completely ignore these (they're not written to the output) */
ds@1952
   245
            NeedToken(LexToken.TOKEN_SYMBOL);
pred@2019
   246
            w.structList.Add(LexString);
ds@1952
   247
            NeedChar('{');
ds@1952
   248
ds@1952
   249
            bool need_closebrace = true;
ds@1952
   250
            bool more_entries = true;
ds@1952
   251
            do
ds@1952
   252
            {
ds@1952
   253
                if (MatchChar('}'))
ds@1952
   254
                {
ds@1952
   255
                    need_closebrace = false;
ds@1952
   256
                    break;
ds@1952
   257
                }
ds@1952
   258
                MatchToken(LexToken.TOKEN_CONST);
ds@1952
   259
                MatchToken(LexToken.TOKEN_LABEL);
ds@1952
   260
                NeedToken(LexToken.TOKEN_SYMBOL);
ds@1952
   261
                while (MatchChar('['))
ds@1952
   262
                {
ds@1952
   263
                    DiscardUntilChar(']');
ds@1952
   264
                    NeedChar(']');
ds@1952
   265
                }
ds@1952
   266
                more_entries = MatchChar(',');
ds@1952
   267
                MatchToken(LexToken.TOKEN_DOCBLOCK);
ds@1952
   268
                if (!more_entries)
ds@1952
   269
                {
ds@1952
   270
                    break;
ds@1952
   271
                }
ds@1952
   272
            } while (true);
ds@1952
   273
            if (need_closebrace)
ds@1952
   274
            {
ds@1952
   275
                NeedChar('}');
ds@1952
   276
            }
ds@1952
   277
            NeedChar(';');
ds@1952
   278
        }
ds@1952
   279
ds@1952
   280
        private void PARSE_Include(ParseWriter w)
ds@1952
   281
        {
ds@1952
   282
            /* read until the end of the line */
ds@1952
   283
            int index = Contents.ToString().IndexOf('\n');
ds@1952
   284
            string value;
ds@1952
   285
            if (index == -1)
ds@1952
   286
            {
ds@1952
   287
                value = Contents.ToString();
ds@1952
   288
                Contents.Remove(0, value.Length);
ds@1952
   289
            }
ds@1952
   290
            else if (index == 0)
ds@1952
   291
            {
ds@1952
   292
                value = "";
ds@1952
   293
            }
ds@1952
   294
            else
ds@1952
   295
            {
ds@1952
   296
                value = Contents.ToString().Substring(0, index);
ds@1952
   297
                Contents.Remove(0, index);
ds@1952
   298
            }
ds@1952
   299
ds@1952
   300
            /* Strip whitespace */
ds@1952
   301
            value = value.Trim('\r', '\t', ' ');
ds@1952
   302
ds@1952
   303
            /* Write */
ds@1952
   304
            w.BeginSection("include");
ds@1952
   305
            w.WritePair("name", value);
ds@1952
   306
            w.EndSection();
ds@1952
   307
        }
ds@1952
   308
ds@1952
   309
        private void PARSE_FuncEnum(ParseWriter w)
ds@1952
   310
        {
ds@1952
   311
            w.BeginSection("funcenum");
ds@1952
   312
ds@1952
   313
            /* Get the functag name */
ds@1952
   314
            NeedToken(LexToken.TOKEN_SYMBOL);
ds@1952
   315
            w.WritePair("name", LexString);
pred@2019
   316
            w.funcenumList.Add(LexString);
ds@1952
   317
ds@1952
   318
            NeedChar('{');
ds@1952
   319
ds@1952
   320
            bool need_closebrace = true;
ds@1952
   321
            do
ds@1952
   322
            {
ds@1952
   323
                /* Shortcut out? */
ds@1952
   324
                if (MatchChar('}'))
ds@1952
   325
                {
ds@1952
   326
                    need_closebrace = false;
ds@1952
   327
                    break;
ds@1952
   328
                }
ds@1952
   329
ds@1952
   330
                /* Start function section */
ds@1952
   331
                w.BeginSection("function");
ds@1952
   332
ds@1952
   333
                if (MatchToken(LexToken.TOKEN_DOCBLOCK))
ds@1952
   334
                {
ds@1952
   335
                    w.WritePair("doc", LexString);
ds@1952
   336
                }
ds@1952
   337
                
ds@1952
   338
                /* Get the return tag */
ds@1952
   339
                if (MatchToken(LexToken.TOKEN_LABEL))
ds@1952
   340
                {
ds@1952
   341
                    w.WritePair("return", LexString);
ds@1952
   342
                }
ds@1952
   343
ds@1952
   344
                /* Get function type */
ds@1952
   345
                if (MatchToken(LexToken.TOKEN_PUBLIC))
ds@1952
   346
                {
ds@1952
   347
                    w.WritePair("type", "public");
ds@1952
   348
                }
ds@1952
   349
                else if (MatchToken(LexToken.TOKEN_STOCK))
ds@1952
   350
                {
ds@1952
   351
                    w.WritePair("type", "stock");
ds@1952
   352
                }
ds@1952
   353
                
ds@1952
   354
                /* Parse the parameters */
ds@1952
   355
                ParseParameters(w);
ds@1952
   356
ds@1952
   357
                /* End the section */
ds@1952
   358
                w.EndSection();
ds@1952
   359
ds@1952
   360
                if (!MatchChar(','))
ds@1952
   361
                {
ds@1952
   362
                    break;
ds@1952
   363
                }
ds@1952
   364
            } while (true);
ds@1952
   365
ds@1952
   366
            if (need_closebrace)
ds@1952
   367
            {
ds@1952
   368
                NeedChar('}');
ds@1952
   369
            }
ds@1952
   370
            NeedChar(';');
ds@1952
   371
ds@1952
   372
            w.EndSection();
ds@1952
   373
        }
ds@1952
   374
ds@1952
   375
        private void PARSE_FuncTag(ParseWriter w)
ds@1952
   376
        {
ds@1952
   377
            w.BeginSection("functag");
ds@1952
   378
ds@1952
   379
            /* Get the functag name */
ds@1952
   380
            NeedToken(LexToken.TOKEN_SYMBOL);
ds@1952
   381
            w.WritePair("name", LexString);
pred@2019
   382
            w.functagList.Add(LexString);
ds@1952
   383
ds@1952
   384
            if (MatchToken(LexToken.TOKEN_LABEL))
ds@1952
   385
            {
ds@1952
   386
                w.WritePair("return", LexString);
ds@1952
   387
            }
ds@1952
   388
ds@1952
   389
            if (MatchToken(LexToken.TOKEN_PUBLIC))
ds@1952
   390
            {
ds@1952
   391
                w.WritePair("type", "public");
ds@1952
   392
            }
ds@1952
   393
            else if (MatchToken(LexToken.TOKEN_STOCK))
ds@1952
   394
            {
ds@1952
   395
                w.WritePair("type", "stock");
ds@1952
   396
            }
ds@1952
   397
ds@1952
   398
            ParseParameters(w);
ds@1952
   399
            NeedChar(';');
ds@1952
   400
            
ds@1952
   401
            w.EndSection();
ds@1952
   402
        }
ds@1952
   403
ds@1952
   404
        private void PARSE_Function(LexToken tok, ParseWriter w)
ds@1952
   405
        {
ds@1952
   406
            string tag="", name;
ds@1952
   407
ds@1952
   408
            /* Get the return value */
ds@1952
   409
            if (MatchToken(LexToken.TOKEN_LABEL))
ds@1952
   410
            {
ds@1952
   411
                tag = LexString;
ds@1952
   412
            }
ds@1952
   413
ds@1952
   414
            if (MatchToken(LexToken.TOKEN_OPERATOR))
ds@1952
   415
            {
ds@1952
   416
                /* Check if we're some sort of invalid function */
ds@1952
   417
                DiscardUntilCharOrComment('(', false);
ds@1952
   418
                ParseParameters(null);
ds@1952
   419
                if (tok == LexToken.TOKEN_STOCK)
ds@1952
   420
                {
ds@1952
   421
                    ignore_block();
ds@1952
   422
                }
ds@1952
   423
                else if (tok == LexToken.TOKEN_NATIVE || tok == LexToken.TOKEN_FORWARD)
ds@1952
   424
                {
ds@1952
   425
                    if (MatchChar('='))
ds@1952
   426
                    {
ds@1952
   427
                        DiscardUntilCharOrComment(';', false);
ds@1952
   428
                    }
ds@1952
   429
                    NeedChar(';');
ds@1952
   430
                }
ds@1952
   431
                return;
ds@1952
   432
            }
ds@1952
   433
ds@1952
   434
            /* Get the name */
ds@1952
   435
            NeedToken(LexToken.TOKEN_SYMBOL);
ds@1952
   436
            name = LexString;
ds@1952
   437
ds@1952
   438
            PARSE_Function(tok, w, tag, name);
ds@1952
   439
        }
ds@1952
   440
ds@1952
   441
        private void PARSE_Function(LexToken tok, ParseWriter w, string tag, string name)
ds@1952
   442
        {
ds@1952
   443
            if (tok == LexToken.TOKEN_FORWARD)
ds@1952
   444
            {
ds@1952
   445
                w.BeginSection("forward");
pred@2019
   446
                w.forwardList.Add(name);
ds@1952
   447
            }
ds@1952
   448
            else if (tok == LexToken.TOKEN_NATIVE)
ds@1952
   449
            {
ds@1952
   450
                w.BeginSection("native");
pred@2019
   451
                w.nativeList.Add(name);
ds@1952
   452
            }
ds@1952
   453
            else if (tok == LexToken.TOKEN_STOCK)
ds@1952
   454
            {
ds@1952
   455
                w.BeginSection("stock");
pred@2019
   456
                w.stockList.Add(name);
ds@1952
   457
            }
ds@1952
   458
ds@1952
   459
            w.WritePair("name", name);
ds@1952
   460
ds@1952
   461
            if (tag.Length > 0)
ds@1952
   462
            {
ds@1952
   463
                w.WritePair("return", tag);
ds@1952
   464
            }
ds@1952
   465
ds@1952
   466
            ParseParameters(w);
ds@1952
   467
ds@1952
   468
            if (tok == LexToken.TOKEN_STOCK)
ds@1952
   469
            {
ds@1952
   470
                ignore_block();
ds@1952
   471
            }
ds@1952
   472
            else
ds@1952
   473
            {
ds@1952
   474
                /* Make sure there's a semicolon */
ds@1952
   475
                if (MatchChar('='))
ds@1952
   476
                {
ds@1952
   477
                    DiscardUntilCharOrComment(';', false);
ds@1952
   478
                }
ds@1952
   479
                NeedChar(';');
ds@1952
   480
            }
ds@1952
   481
ds@1952
   482
            w.EndSection();
ds@1952
   483
        }
ds@1952
   484
ds@1952
   485
        private void ParseParameters(ParseWriter w)
ds@1952
   486
        {
ds@1952
   487
            NeedChar('(');
ds@1952
   488
            string extra;
ds@1952
   489
            do
ds@1952
   490
            {
ds@1952
   491
                if (MatchChar(')'))
ds@1952
   492
                {
ds@1952
   493
                    return;
ds@1952
   494
                }
ds@1952
   495
ds@1952
   496
                /* Start the parameter */
ds@1952
   497
                if (w != null)
ds@1952
   498
                {
ds@1952
   499
                    w.BeginSection("parameter");
ds@1952
   500
                }
ds@1952
   501
ds@1952
   502
                /* Check for a 'const' token */
ds@1952
   503
                if (MatchToken(LexToken.TOKEN_CONST) && w != null)
ds@1952
   504
                {
ds@1952
   505
                    w.WritePair("const", "true");
ds@1952
   506
                }
ds@1952
   507
ds@1952
   508
                /* Check if it's by reference */
ds@1952
   509
                if (MatchChar('&') && w != null)
ds@1952
   510
                {
ds@1952
   511
                    w.WritePair("byref", "true");
ds@1952
   512
                }
ds@1952
   513
ds@1952
   514
                /* Check for a tag */
ds@1952
   515
                if (MatchToken(LexToken.TOKEN_LABEL) && w != null)
ds@1952
   516
                {
ds@1952
   517
                    w.WritePair("tag", LexString);
ds@1952
   518
                }
ds@1952
   519
ds@1952
   520
                /* Get the symbol and write it */
ds@1952
   521
                if (MatchString(0, "...") && w != null)
ds@1952
   522
                {
ds@1952
   523
                    w.WritePair("name", "...");
ds@1952
   524
                }
ds@1952
   525
                else
ds@1952
   526
                {
ds@1952
   527
                    NeedToken(LexToken.TOKEN_SYMBOL);
ds@1952
   528
                    if (w != null)
ds@1952
   529
                    {
ds@1952
   530
                        w.WritePair("name", LexString);
ds@1952
   531
                    }
ds@1952
   532
                }
ds@1952
   533
ds@1952
   534
                /* Check if we have a default value yet */
ds@1952
   535
                bool default_value = false;
ds@1952
   536
                if (MatchChar('='))
ds@1952
   537
                {
ds@1952
   538
                    default_value = true;
ds@1952
   539
                }
ds@1952
   540
                else if (MatchChar('['))
ds@1952
   541
                {
ds@1952
   542
                    extra = "";
ds@1952
   543
                    string temp;
ds@1952
   544
                    do
ds@1952
   545
                    {
ds@1952
   546
                        extra += "[";
ds@1952
   547
                        temp = ConsumeUntilChar(']');
ds@1952
   548
                        if (temp != null)
ds@1952
   549
                        {
ds@1952
   550
                            extra += temp;
ds@1952
   551
                        }
ds@1952
   552
                        extra += "]";
ds@1952
   553
                        NeedChar(']');
ds@1952
   554
                    } while (MatchChar('['));
ds@1952
   555
                    w.WritePair("decoration", extra);
ds@1952
   556
                }
ds@1952
   557
ds@1952
   558
                /* If we have a default value, get it */
ds@1952
   559
                if (default_value || MatchChar('='))
ds@1952
   560
                {
ds@1952
   561
                    if ((extra = ConsumeParamDefValue()) == null)
ds@1952
   562
                    {
ds@1952
   563
                        throw new ParseException("Expected a default value; found none");
ds@1952
   564
                    }
ds@1952
   565
                    if (w != null)
ds@1952
   566
                    {
ds@1952
   567
                        w.WritePair("defval", extra);
ds@1952
   568
                    }
ds@1952
   569
                }
ds@1952
   570
ds@1952
   571
                /* End the parameter */
ds@1952
   572
                if (w != null)
ds@1952
   573
                {
ds@1952
   574
                    w.EndSection();
ds@1952
   575
                }
ds@1952
   576
ds@1952
   577
                if (!MatchChar(','))
ds@1952
   578
                {
ds@1952
   579
                    break;
ds@1952
   580
                }
ds@1952
   581
            } while (true);
ds@1952
   582
ds@1952
   583
            NeedChar(')');
ds@1952
   584
        }
ds@1952
   585
ds@1952
   586
        private void PARSE_Enum(ParseWriter w)
ds@1952
   587
        {
ds@1952
   588
            string name = "";
ds@1952
   589
            if (MatchToken(LexToken.TOKEN_SYMBOL))
ds@1952
   590
            {
ds@1952
   591
                name = LexString;
ds@1952
   592
            }
ds@1952
   593
ds@1952
   594
            w.BeginSection("enum");
ds@1952
   595
            w.WritePair("name", name);
ds@1952
   596
            w.BeginSection("properties");
ds@1952
   597
pred@2019
   598
            w.enumList.Add(name);
pred@2019
   599
ds@1952
   600
            NeedChar('{');
ds@1952
   601
            bool more_entries = true;
ds@1952
   602
            bool need_closebrace = true;
pred@2019
   603
pred@2019
   604
            if (MatchToken(LexToken.TOKEN_DOCBLOCK))
pred@2019
   605
            {
pred@2019
   606
              w.WritePair("doc", LexString);
pred@2019
   607
            }
pred@2019
   608
ds@1952
   609
            do
ds@1952
   610
            {
ds@1952
   611
                if (MatchChar('}'))
ds@1952
   612
                {
ds@1952
   613
                    need_closebrace = false;
ds@1952
   614
                    break;
ds@1952
   615
                }
ds@1952
   616
                NeedToken(LexToken.TOKEN_SYMBOL);
ds@1952
   617
                name = LexString;
pred@2019
   618
 
ds@1952
   619
                if (MatchChar('='))
ds@1952
   620
                {
pred@2019
   621
                    DiscardUntilCharOrComment(",\n}", true);
ds@1952
   622
                }
ds@1952
   623
                more_entries = MatchChar(',');
ds@1952
   624
                w.WritePair("name", name);
pred@2019
   625
                w.enumTypeList.Add(name);
ds@1952
   626
                if (MatchToken(LexToken.TOKEN_DOCBLOCK))
ds@1952
   627
                {
ds@1952
   628
                    w.WritePair("doc", LexString);
ds@1952
   629
                }
ds@1952
   630
            } while (more_entries);
ds@1952
   631
            if (need_closebrace)
ds@1952
   632
            {
ds@1952
   633
                NeedChar('}');
ds@1952
   634
            }
ds@1952
   635
            NeedChar(';');
ds@1952
   636
            w.EndSection();
ds@1952
   637
            w.EndSection();
ds@1952
   638
        }
ds@1952
   639
ds@1952
   640
        private void PARSE_Define(ParseWriter w)
ds@1952
   641
        {
ds@1952
   642
            /* first, get the symbol name */
ds@1952
   643
            NeedToken(LexToken.TOKEN_SYMBOL);
ds@1952
   644
            string name = LexString;
ds@1952
   645
            string value;
ds@1952
   646
ds@1952
   647
            /* read until we hit the end of a line OR a comment */
ds@1952
   648
            int endpoint = -1;
ds@1952
   649
            for (int i = 0; i < Contents.Length; i++)
ds@1952
   650
            {
ds@1952
   651
                if (Contents[i] == '\n')
ds@1952
   652
                {
ds@1952
   653
                    endpoint = i;
ds@1952
   654
                    break;
ds@1952
   655
                }
ds@1952
   656
                else if (CheckString(i, "/**"))
ds@1952
   657
                {
ds@1952
   658
                    endpoint = i;
ds@1952
   659
                    break;
ds@1952
   660
                }
ds@1952
   661
            }
ds@1952
   662
ds@1952
   663
            if (endpoint < 1)
ds@1952
   664
            {
ds@1952
   665
                value = "";
ds@1952
   666
            }
ds@1952
   667
            else
ds@1952
   668
            {
ds@1952
   669
                value = Contents.ToString().Substring(0, endpoint);
ds@1952
   670
                Contents.Remove(0, endpoint);
ds@1952
   671
            }
ds@1952
   672
ds@1952
   673
            /* Write */
pred@2019
   674
            w.defineList.Add(name);
pred@2019
   675
ds@1952
   676
            w.BeginSection("define");
ds@1952
   677
            w.WritePair("name", name);
ds@1952
   678
            w.WritePair("linetext", value);
ds@1952
   679
            w.EndSection();
ds@1952
   680
        }
ds@1952
   681
ds@1952
   682
        /* Reads up to a character and discards all text before it.
ds@1952
   683
         */
ds@1952
   684
        private void DiscardUntilChar(char c)
ds@1952
   685
        {
ds@1952
   686
            int eol = Contents.ToString().IndexOf(c);
ds@1952
   687
ds@1952
   688
            if (eol < 1)
ds@1952
   689
            {
ds@1952
   690
                return;
ds@1952
   691
            }
ds@1952
   692
ds@1952
   693
            Contents.Remove(0, eol);
ds@1952
   694
        }
ds@1952
   695
ds@1952
   696
        /* Same as DiscardUntilChar, except the discarded
ds@1952
   697
         * text is returned first.
ds@1952
   698
         */
ds@1952
   699
        private string ConsumeUntilChar(char c)
ds@1952
   700
        {
ds@1952
   701
            int eol = Contents.ToString().IndexOf(c);
ds@1952
   702
ds@1952
   703
            if (eol < 1)
ds@1952
   704
            {
ds@1952
   705
                return null;
ds@1952
   706
            }
ds@1952
   707
ds@1952
   708
            string rest = Contents.ToString().Substring(0, eol);
ds@1952
   709
            Contents.Remove(0, eol);
ds@1952
   710
            return rest;
ds@1952
   711
        }
ds@1952
   712
ds@1952
   713
        /**
ds@1952
   714
         * Returns a parameter's default value.
ds@1952
   715
         */
ds@1952
   716
        private string ConsumeParamDefValue()
ds@1952
   717
        {
ds@1952
   718
            int eol = FindEndOfExprLine(0);
ds@1952
   719
            if (eol < 1)
ds@1952
   720
            {
ds@1952
   721
                return null;
ds@1952
   722
            }
ds@1952
   723
ds@1952
   724
            string rest = Contents.ToString().Substring(0, eol);
ds@1952
   725
            Contents.Remove(0, eol);
ds@1952
   726
            return rest;
ds@1952
   727
        }
ds@1952
   728
ds@1952
   729
        /**
ds@1952
   730
         * Discards all data up to a certain character, ignoring
ds@1952
   731
         * the contents, unless specified to stop at a doc block comment.
ds@1952
   732
         */
ds@1952
   733
        private void DiscardUntilCharOrComment(char c, bool stop_at_comment)
ds@1952
   734
        {
ds@1952
   735
            for (int i = 0; i < Contents.Length; i++)
ds@1952
   736
            {
ds@1952
   737
                if (Contents[i] == '\n')
ds@1952
   738
                {
ds@1952
   739
                    LineNo++;
ds@1952
   740
                }
ds@1952
   741
                else if (Contents[i] == c)
ds@1952
   742
                {
ds@1952
   743
                    if (i > 0)
ds@1952
   744
                    {
ds@1952
   745
                        Contents.Remove(0, i);
ds@1952
   746
                    }
ds@1952
   747
                    return;
ds@1952
   748
                }
ds@1952
   749
                else if (stop_at_comment && CheckString(i, "/**"))
ds@1952
   750
                {
ds@1952
   751
                    if (i > 0)
ds@1952
   752
                    {
ds@1952
   753
                        Contents.Remove(0, i);
ds@1952
   754
                    }
ds@1952
   755
                    return;
ds@1952
   756
                }
ds@1952
   757
            }
ds@1952
   758
        }
ds@1952
   759
pred@2019
   760
       /**
pred@2019
   761
        * Discards all data up to a certain character, ignoring
pred@2019
   762
        * the contents, unless specified to stop at a doc block comment.
pred@2019
   763
        */
pred@2019
   764
        private void DiscardUntilCharOrComment(string s, bool stop_at_comment)
pred@2019
   765
        {
pred@2019
   766
          for (int i = 0; i < Contents.Length; i++)
pred@2019
   767
          {
pred@2019
   768
            if (Contents[i] == '\n')
pred@2019
   769
            {
pred@2019
   770
              LineNo++;
pred@2019
   771
            }
pred@2019
   772
            else if (s.Contains(Contents[i].ToString()))
pred@2019
   773
            {
pred@2019
   774
              if (i > 0)
pred@2019
   775
              {
pred@2019
   776
                Contents.Remove(0, i);
pred@2019
   777
              }
pred@2019
   778
              return;
pred@2019
   779
            }
pred@2019
   780
            else if (stop_at_comment && CheckString(i, "/**"))
pred@2019
   781
            {
pred@2019
   782
              if (i > 0)
pred@2019
   783
              {
pred@2019
   784
                Contents.Remove(0, i);
pred@2019
   785
              }
pred@2019
   786
              return;
pred@2019
   787
            }
pred@2019
   788
          }
pred@2019
   789
        }
pred@2019
   790
ds@1952
   791
        /**
ds@1952
   792
         * Finds the end of an expression
ds@1952
   793
         */
ds@1952
   794
        private int FindEndOfExprLine(int start)
ds@1952
   795
        {
ds@1952
   796
            for (int i = start; i < Contents.Length; i++)
ds@1952
   797
            {
ds@1952
   798
                if (Contents[i] == '{')
ds@1952
   799
                {
ds@1952
   800
                    if ((i = FindEndOfConsumptionLine('}', start+1)) == -1)
ds@1952
   801
                    {
ds@1952
   802
                        throw new ParseException("Expected token: } (found none)");
ds@1952
   803
                    }
ds@1952
   804
                }
ds@1952
   805
                else if (Contents[i] == ')' || Contents[i] == ',')
ds@1952
   806
                {
ds@1952
   807
                    return i;
ds@1952
   808
                }
ds@1952
   809
            }
ds@1952
   810
ds@1952
   811
            return -1;
ds@1952
   812
        }
ds@1952
   813
ds@1952
   814
        private int FindEndOfConsumptionLine(char c, int start)
ds@1952
   815
        {
ds@1952
   816
            for (int i = start; i < Contents.Length; i++)
ds@1952
   817
            {
ds@1952
   818
                if (Contents[i] == '{')
ds@1952
   819
                {
ds@1952
   820
                    if ((i = FindEndOfConsumptionLine('}', i + 1)) == -1)
ds@1952
   821
                    {
ds@1952
   822
                        throw new ParseException("Expected token: } (found none)");
ds@1952
   823
                    }
ds@1952
   824
                }
ds@1952
   825
                else if (Contents[i] == c)
ds@1952
   826
                {
ds@1952
   827
                    return i;
ds@1952
   828
                }
ds@1952
   829
            }
ds@1952
   830
ds@1952
   831
            return -1;
ds@1952
   832
        }
ds@1952
   833
ds@1952
   834
        private void Initialize()
ds@1952
   835
        {
ds@1952
   836
            /* Open file, read it, and close it */
ds@1952
   837
            StreamReader sr = File.OpenText(FileName);
ds@1952
   838
            Contents = new StringBuilder(sr.ReadToEnd());
ds@1952
   839
            sr.Close();
ds@1952
   840
ds@1952
   841
            /* Strip comments (may throw an exception) */
ds@1952
   842
            preprocess();
ds@1952
   843
ds@1952
   844
            /* Lastly, initialize the tokenizer */
ds@1952
   845
            Tokenizer.Initialize();
ds@1952
   846
        }
ds@1952
   847
ds@1952
   848
        /* Consumes a token iff it matches */
ds@1952
   849
        private bool MatchToken(LexToken token)
ds@1952
   850
        {
ds@1952
   851
            LexToken tok = lex();
ds@1952
   852
            if (tok != token)
ds@1952
   853
            {
ds@1952
   854
                lexpush();
ds@1952
   855
                return false;
ds@1952
   856
            }
ds@1952
   857
            return true;
ds@1952
   858
        }
ds@1952
   859
ds@1952
   860
        /* Consumes a token and aborts the parser if there is no match */
ds@1952
   861
        private void NeedToken(LexToken token)
ds@1952
   862
        {
ds@1952
   863
            if (!MatchToken(token))
ds@1952
   864
            {
ds@1952
   865
                throw new ParseException("Expected token: " + token);
ds@1952
   866
            }
ds@1952
   867
        }
ds@1952
   868
ds@1952
   869
        /* Consumes a character iff it matches */
ds@1952
   870
        private bool MatchChar(char c)
ds@1952
   871
        {
ds@1952
   872
            if (lex() != LexToken.TOKEN_CHARACTER)
ds@1952
   873
            {
ds@1952
   874
                lexpush();
ds@1952
   875
                return false;
ds@1952
   876
            }
ds@1952
   877
            if (_LexChar != c)
ds@1952
   878
            {
ds@1952
   879
                lexpush();
ds@1952
   880
                return false;
ds@1952
   881
            }
ds@1952
   882
            return true;
ds@1952
   883
        }
ds@1952
   884
ds@1952
   885
        /* Consumes a character or aborts the parser if there is no match */
ds@1952
   886
        private void NeedChar(char c)
ds@1952
   887
        {
ds@1952
   888
            if (!MatchChar(c))
ds@1952
   889
            {
ds@1952
   890
                throw new ParseException("Expected character: " + c);
ds@1952
   891
            }
ds@1952
   892
        }
ds@1952
   893
        
ds@1952
   894
        /* Saves the lexer state so we don't reparse unnecessarily */
ds@1952
   895
        private void lexpush()
ds@1952
   896
        {
ds@1952
   897
            if (LexPushed)
ds@1952
   898
            {
ds@1952
   899
                throw new ParseException("Lexer cannot be pushed twice");
ds@1952
   900
            }
ds@1952
   901
            LexPushed = true;
ds@1952
   902
        }
ds@1952
   903
ds@1952
   904
        /* Attempts to match a string against the next stream tokens.
ds@1952
   905
         * On success, the token stream is advanced.
ds@1952
   906
         */
ds@1952
   907
        private bool MatchString(int startPos, string str)
ds@1952
   908
        {
ds@1952
   909
            if (str.Length > Contents.Length - startPos)
ds@1952
   910
            {
ds@1952
   911
                return false;
ds@1952
   912
            }
ds@1952
   913
            for (int i = 0; i < str.Length; i++)
ds@1952
   914
            {
ds@1952
   915
                if (str[i] != Contents[startPos + i])
ds@1952
   916
                {
ds@1952
   917
                    return false;
ds@1952
   918
                }
ds@1952
   919
            }
ds@1952
   920
            Contents.Remove(startPos, str.Length);
ds@1952
   921
            return true;
ds@1952
   922
        }
ds@1952
   923
ds@1952
   924
        /* Attempts to match a string against the next stream tokens */
ds@1952
   925
        private bool CheckString(int startPos, string str)
ds@1952
   926
        {
ds@1952
   927
            if (str.Length > Contents.Length - startPos)
ds@1952
   928
            {
ds@1952
   929
                return false;
ds@1952
   930
            }
ds@1952
   931
            for (int i = 0; i < str.Length; i++)
ds@1952
   932
            {
ds@1952
   933
                if (str[i] != Contents[startPos + i])
ds@1952
   934
                {
ds@1952
   935
                    return false;
ds@1952
   936
                }
ds@1952
   937
            }
ds@1952
   938
            return true;
ds@1952
   939
        }
ds@1952
   940
ds@1952
   941
        /* Ignores a block starting and ending with { and } respectively */
ds@1952
   942
        private void ignore_block()
ds@1952
   943
        {
ds@1952
   944
            /* Hrm, this is tricky.  First, expect an open brace. */
ds@1952
   945
            NeedChar('{');
ds@1952
   946
            /* Consume the entire function by keeping track of braces */
ds@1952
   947
            int numBraces = 1;
ds@1952
   948
            bool in_string = false;
ds@1952
   949
            bool found_end = false;
ds@1952
   950
            for (int i = 0; i < Contents.Length; i++)
ds@1952
   951
            {
ds@1952
   952
                if (Contents[i] == '\n')
ds@1952
   953
                {
ds@1952
   954
                    LineNo++;
ds@1952
   955
                }
ds@1952
   956
                if (!in_string)
ds@1952
   957
                {
ds@1952
   958
                    if (Contents[i] == '"')
ds@1952
   959
                    {
ds@1952
   960
                        in_string = true;
ds@1952
   961
                    }
ds@1952
   962
                    else if (Contents[i] == '{')
ds@1952
   963
                    {
ds@1952
   964
                        numBraces++;
ds@1952
   965
                    }
ds@1952
   966
                    else if (Contents[i] == '}')
ds@1952
   967
                    {
ds@1952
   968
                        if (numBraces == 0)
ds@1952
   969
                        {
ds@1952
   970
                            throw new ParseException("Unable to find a matching open brace");
ds@1952
   971
                        }
ds@1952
   972
                        if (--numBraces == 0)
ds@1952
   973
                        {
ds@1952
   974
                            Contents.Remove(0, i + 1);
ds@1952
   975
                            found_end = true;
ds@1952
   976
                            break;
ds@1952
   977
                        }
ds@1952
   978
                    }
ds@1952
   979
                }
ds@1952
   980
                else
ds@1952
   981
                {
ds@1952
   982
                    /* It is possible to be in a string and at the first character */
ds@1952
   983
                    if (Contents[i] == '"' && Contents[i - 1] != '\\')
ds@1952
   984
                    {
ds@1952
   985
                        in_string = false;
ds@1952
   986
                    }
ds@1952
   987
                }
ds@1952
   988
            }
ds@1952
   989
            if (!found_end)
ds@1952
   990
            {
ds@1952
   991
                throw new ParseException("Unable to find a matching close brace");
ds@1952
   992
            }
ds@1952
   993
        }
ds@1952
   994
ds@1952
   995
        /* Strips all non-doc C and C++ style comments. as well as unused pragmas */
ds@1952
   996
        private void preprocess()
ds@1952
   997
        {
ds@1952
   998
            if (Contents.Length == 1)
ds@1952
   999
            {
ds@1952
  1000
                return;
ds@1952
  1001
            }
ds@1952
  1002
ds@1952
  1003
            bool in_comment = false;
ds@1952
  1004
            bool is_ml = false;
ds@1952
  1005
            bool is_doc = false;
ds@1952
  1006
            bool is_newline = true;
ds@1952
  1007
pred@2019
  1008
            for (int i=0; i<Contents.Length; i++)
ds@1952
  1009
            {
ds@1952
  1010
                if (Contents[i] == '\n')
ds@1952
  1011
                {
ds@1952
  1012
                    is_newline = true;
ds@1952
  1013
                }
ds@1952
  1014
                if (!in_comment)
ds@1952
  1015
                {
ds@1952
  1016
                    if (Contents[i] == '/')
ds@1952
  1017
                    {
ds@1952
  1018
                        if (Contents[i+1] == '/')
ds@1952
  1019
                        {
ds@1952
  1020
                            in_comment = true;
ds@1952
  1021
                            is_ml = false;
ds@1952
  1022
                            is_doc = false;
pred@2019
  1023
                            is_newline = false;
ds@1952
  1024
                        } else if (Contents[i+1] == '*') {
ds@1952
  1025
                            /* Detect whether this is a doc comment or not */
ds@1952
  1026
                            if (Contents.Length - i >= 3
ds@1952
  1027
                                && Contents[i+2] == '*')
ds@1952
  1028
                            {
ds@1952
  1029
                                is_doc = true;
ds@1952
  1030
                            } else {
ds@1952
  1031
                                is_doc = false;
ds@1952
  1032
                            }
ds@1952
  1033
                            in_comment = true;
ds@1952
  1034
                            is_ml = true;
ds@1952
  1035
                        }
ds@1952
  1036
                        if (in_comment && !is_doc)
ds@1952
  1037
                        {
ds@1952
  1038
                            Contents[i] = ' ';
ds@1952
  1039
                            Contents[i+1] = ' ';        /* Just for safety */
ds@1952
  1040
                        }
ds@1952
  1041
                    }
ds@1952
  1042
                    else if (is_newline)
ds@1952
  1043
                    {
ds@1952
  1044
                        if (Contents[i] == '#')
ds@1952
  1045
                        {
ds@1952
  1046
                            if (!CheckString(i, "#define")
ds@1952
  1047
                                && !CheckString(i, "#include"))
ds@1952
  1048
                            {
ds@1952
  1049
                                /* Ignore! */
ds@1952
  1050
                                Contents[i] = ' ';
ds@1952
  1051
                                in_comment = true;
ds@1952
  1052
                                is_doc = false;
ds@1952
  1053
                                is_ml = false;
ds@1952
  1054
                            }
ds@1952
  1055
                            is_newline = false;
ds@1952
  1056
                        }
ds@1952
  1057
                        else if (!Char.IsWhiteSpace(Contents[i]))
ds@1952
  1058
                        {
ds@1952
  1059
                            is_newline = false;
ds@1952
  1060
                        }
ds@1952
  1061
                    }
ds@1952
  1062
                } else {
ds@1952
  1063
                    if (is_ml)
ds@1952
  1064
                    {
ds@1952
  1065
                        if (Contents[i] == '*' && Contents[i+1] == '/')
ds@1952
  1066
                        {
ds@1952
  1067
                            if (!is_doc)
ds@1952
  1068
                            {
ds@1952
  1069
                                Contents[i] = ' ';
ds@1952
  1070
                                Contents[i+1] = ' ';
ds@1952
  1071
                            }
ds@1952
  1072
                            in_comment = false;
ds@1952
  1073
                            i++;    /* Don't look at the next character */
ds@1952
  1074
                        }
ds@1952
  1075
                        else if (!is_doc && Contents[i] != '\n')
ds@1952
  1076
                        {
ds@1952
  1077
                            /* Erase comment */
ds@1952
  1078
                            Contents[i] = ' ';
ds@1952
  1079
                        }
ds@1952
  1080
                    } else {
ds@1952
  1081
                        if (is_newline)
ds@1952
  1082
                        {
ds@1952
  1083
                            in_comment = false;
ds@1952
  1084
                        }
ds@1952
  1085
                        else
ds@1952
  1086
                        {
ds@1952
  1087
                            /* Erase comment */
ds@1952
  1088
                            Contents[i] = ' ';
ds@1952
  1089
                        }
ds@1952
  1090
                    }
ds@1952
  1091
                }
ds@1952
  1092
            }
ds@1952
  1093
ds@1952
  1094
            if (in_comment)
ds@1952
  1095
            {
ds@1952
  1096
                throw new ParseException("Multi-line comment has no terminator");
ds@1952
  1097
            }
ds@1952
  1098
        }
ds@1952
  1099
ds@1952
  1100
        /* Strips whitespace */
ds@1952
  1101
        private void trim_whitespace_pre()
ds@1952
  1102
        {
ds@1952
  1103
            int spaces = 0;
ds@1952
  1104
            while (spaces < Contents.Length
ds@1952
  1105
                   && Contents[spaces] != '\0'
ds@1952
  1106
                   && Char.IsWhiteSpace(Contents[spaces]))
ds@1952
  1107
            {
ds@1952
  1108
                if (Contents[spaces] == '\n')
ds@1952
  1109
                {
ds@1952
  1110
                    LineNo++;
ds@1952
  1111
                }
ds@1952
  1112
                spaces++;
ds@1952
  1113
            }
ds@1952
  1114
            if (spaces > 0)
ds@1952
  1115
            {
ds@1952
  1116
                Contents.Remove(0, spaces);
ds@1952
  1117
            }
ds@1952
  1118
        }
ds@1952
  1119
ds@1952
  1120
        private void preprocess_line()
ds@1952
  1121
        {
ds@1952
  1122
            trim_whitespace_pre();
ds@1952
  1123
        }
ds@1952
  1124
ds@1952
  1125
        private bool IsSymbolChar(char c, bool first)
ds@1952
  1126
        {
ds@1952
  1127
            bool valid = (first ? Char.IsLetter(c) : Char.IsLetterOrDigit(c));
ds@1952
  1128
            return (valid || (c == '_'));
ds@1952
  1129
        }
ds@1952
  1130
ds@1952
  1131
        private bool IsEscapedChar(char c)
ds@1952
  1132
        {
ds@1952
  1133
            return (c == 'n'
ds@1952
  1134
                    || c == 'r'
ds@1952
  1135
                    || c == 't'
ds@1952
  1136
                    || c == 'v'
ds@1952
  1137
                    || c == '\\'
ds@1952
  1138
                    || c == '"');
ds@1952
  1139
        }
ds@1952
  1140
ds@1952
  1141
        private bool IsNotKeywordChar(char c)
ds@1952
  1142
        {
ds@1952
  1143
            return (Char.IsWhiteSpace(c)
ds@1952
  1144
                    || (c == '-' || c == '+')
ds@1952
  1145
                    || (c == '/' || c == '*')
ds@1952
  1146
                    || (c == '(' || c == ')')
ds@1952
  1147
                    || (c == '=' || c == '%')
ds@1952
  1148
                    || (c == '>' || c == '<')
ds@1952
  1149
                    || (c == '!' || c == '~')
ds@1952
  1150
                    );
ds@1952
  1151
        }
ds@1952
  1152
ds@1952
  1153
        /* Consumes one lexical token and gathers information about it */
ds@1952
  1154
        private LexToken lex()
ds@1952
  1155
        {
ds@1952
  1156
            if (LexPushed)
ds@1952
  1157
            {
ds@1952
  1158
                LexPushed = false;
ds@1952
  1159
                return _LastToken;
ds@1952
  1160
            }
ds@1952
  1161
ds@1952
  1162
            /* Number of chars we will be deleting from the input */
ds@1952
  1163
            int stripchars = 0;
ds@1952
  1164
ds@1952
  1165
            /* Clear our state */
ds@1952
  1166
            _LastToken = LexToken.TOKEN_NONE;
ds@1952
  1167
ds@1952
  1168
            /* Remove stuff we don't want from the line */
ds@1952
  1169
            preprocess_line();
ds@1952
  1170
ds@1952
  1171
            if (Contents.Length < 1)
ds@1952
  1172
            {
ds@1952
  1173
                _LastToken = LexToken.TOKEN_EOF;
ds@1952
  1174
                return _LastToken;
ds@1952
  1175
            }
ds@1952
  1176
            
ds@1952
  1177
            /* Get the token list */
ds@1952
  1178
            KeywordToken[] tokens = Tokenizer.Tokens;
ds@1952
  1179
            for (int i = 0; i < tokens.Length; i++)
ds@1952
  1180
            {
ds@1952
  1181
                if (!CheckString(0, tokens[i].keyword))
ds@1952
  1182
                {
ds@1952
  1183
                    continue;
ds@1952
  1184
                }
ds@1952
  1185
                int len = tokens[i].keyword.Length;
ds@1952
  1186
                /* Now check to see what the next token is */
ds@1952
  1187
                if (Contents.Length == len
ds@1952
  1188
                    || IsNotKeywordChar(Contents[len]))
ds@1952
  1189
                {
ds@1952
  1190
                    /* We have a token match! */
ds@1952
  1191
                    _LastToken = tokens[i].token;
ds@1952
  1192
                    _LexString = null;
ds@1952
  1193
                    _LexChar = '\0';
ds@1952
  1194
                    stripchars = len;
ds@1952
  1195
                    break;
ds@1952
  1196
                }
ds@1952
  1197
            }
ds@1952
  1198
ds@1952
  1199
            if (_LastToken == LexToken.TOKEN_NONE)
ds@1952
  1200
            {
ds@1952
  1201
                /* See if we can try to read a symbol */
ds@1952
  1202
                if (IsSymbolChar(Contents[0], true))
ds@1952
  1203
                {
ds@1952
  1204
                    int characters = 1;
ds@1952
  1205
                    while (Contents[characters] != '\0'
ds@1952
  1206
                           && IsSymbolChar(Contents[characters], false))
ds@1952
  1207
                    {
ds@1952
  1208
                        characters++;
ds@1952
  1209
                    }
ds@1952
  1210
                    stripchars = characters;
ds@1952
  1211
                    /* We're done! See what's next.. */
ds@1952
  1212
                    if (Contents[characters] == ':')
ds@1952
  1213
                    {
ds@1952
  1214
                        _LastToken = LexToken.TOKEN_LABEL;
ds@1952
  1215
                        stripchars++;
ds@1952
  1216
                    }
ds@1952
  1217
                    else
ds@1952
  1218
                    {
ds@1952
  1219
                        _LastToken = LexToken.TOKEN_SYMBOL;
ds@1952
  1220
                    }
ds@1952
  1221
                    _LexString = Contents.ToString().Substring(0, characters);
ds@1952
  1222
                    _LexChar = _LexString[0];
ds@1952
  1223
                }/*
ds@1952
  1224
                else if (Contents[0] == '"')
ds@1952
  1225
                {
ds@1952
  1226
                    bool reached_end = false;
ds@1952
  1227
                    StringBuilder builder = new StringBuilder();
ds@1952
  1228
                    for (int i = 1; i < Contents.Length; i++)
ds@1952
  1229
                    {
ds@1952
  1230
                        if (i < Contents.Length - 1
ds@1952
  1231
                            && Contents[i] == '\\'
ds@1952
  1232
                            && IsEscapedChar(Contents[i+1]))
ds@1952
  1233
                        {
ds@1952
  1234
                            i++;
ds@1952
  1235
                        }
ds@1952
  1236
                        else if (Contents[i] == '"')
ds@1952
  1237
                        {
ds@1952
  1238
                            reached_end = true;
ds@1952
  1239
                            break;
ds@1952
  1240
                        }
ds@1952
  1241
                        builder.Append(Contents[i]);
ds@1952
  1242
                    }
ds@1952
  1243
                    if (!reached_end)
ds@1952
  1244
                    {
ds@1952
  1245
                        throw new ParseException("Expected end of string; none found");
ds@1952
  1246
                    }
ds@1952
  1247
                    _LexString = builder.ToString();
ds@1952
  1248
                    _LexChar = _LexString.Length > 0 ? _LexString[0] : '\0';
ds@1952
  1249
                }*/
ds@1952
  1250
                else if (Contents[0] == '\'')
ds@1952
  1251
                {
ds@1952
  1252
                    char c = '\0';
ds@1952
  1253
                    if (Contents.Length < 3)
ds@1952
  1254
                    {
ds@1952
  1255
                        throw new ParseException("Expected end of character; none found");
ds@1952
  1256
                    }
ds@1952
  1257
                    else if (Contents[1] == '\\')
ds@1952
  1258
                    {
ds@1952
  1259
                        if (Contents.Length < 4 || !IsEscapedChar(Contents[2]))
ds@1952
  1260
                        {
ds@1952
  1261
                            throw new ParseException("Expected end of character; none found");
ds@1952
  1262
                        }
ds@1952
  1263
                        if (Contents[2] == 'r')
ds@1952
  1264
                        {
ds@1952
  1265
                            c = '\r';
ds@1952
  1266
                        }
ds@1952
  1267
                        else if (Contents[2] == 'n')
ds@1952
  1268
                        {
ds@1952
  1269
                            c = '\n';
ds@1952
  1270
                        }
ds@1952
  1271
                        else if (Contents[2] == 't')
ds@1952
  1272
                        {
ds@1952
  1273
                            c = '\t';
ds@1952
  1274
                        }
ds@1952
  1275
                        else if (Contents[2] == 'v')
ds@1952
  1276
                        {
ds@1952
  1277
                            c = '\v';
ds@1952
  1278
                        }
ds@1952
  1279
                        else if (Contents[2] == '\\')
ds@1952
  1280
                        {
ds@1952
  1281
                            c = '\\';
ds@1952
  1282
                        }
ds@1952
  1283
                        else if (Contents[2] == '"')
ds@1952
  1284
                        {
ds@1952
  1285
                            c = '"';
ds@1952
  1286
                        }
ds@1952
  1287
                        stripchars = 4;
ds@1952
  1288
                    }
ds@1952
  1289
                    else
ds@1952
  1290
                    {
ds@1952
  1291
                        c = Contents[1];
ds@1952
  1292
                        stripchars = 3;
ds@1952
  1293
                    }
ds@1952
  1294
                    _LexString = c.ToString();
ds@1952
  1295
                    _LexChar = c;
ds@1952
  1296
                    _LastToken = LexToken.TOKEN_QUOTCHAR;
ds@1952
  1297
                }
ds@1952
  1298
                else if (CheckString(0, "/**"))
ds@1952
  1299
                {
ds@1952
  1300
                    int endpoint = 0;
ds@1952
  1301
                    for (int i=3; i<Contents.Length-1; i++)
ds@1952
  1302
                    {
ds@1952
  1303
                        if (Contents[i] == '\n')
ds@1952
  1304
                        {
ds@1952
  1305
                            LineNo++;
ds@1952
  1306
                        }
ds@1952
  1307
                        else if (Contents[i] == '*' && Contents[i+1] == '/')
ds@1952
  1308
                        {
ds@1952
  1309
                            endpoint = i+1;
ds@1952
  1310
                            break;
ds@1952
  1311
                        }
ds@1952
  1312
                    }
ds@1952
  1313
                    if (endpoint == 0)
ds@1952
  1314
                    {
ds@1952
  1315
                        throw new ParseException("Expected end of multi-line comment; found end of file");
ds@1952
  1316
                    }
ds@1952
  1317
                    /* We have to factor one more character in because we're zero based */
ds@1952
  1318
                    endpoint++;
ds@1952
  1319
                    _LastToken = LexToken.TOKEN_DOCBLOCK;
ds@1952
  1320
                    _LexString = Contents.ToString().Substring(0, endpoint);
ds@1952
  1321
                    _LexChar = _LexString[0];
ds@1952
  1322
                    stripchars = endpoint;
ds@1952
  1323
                }
ds@1952
  1324
                else
ds@1952
  1325
                {
ds@1952
  1326
                    _LastToken = LexToken.TOKEN_CHARACTER;
ds@1952
  1327
                    _LexString = Contents[0].ToString();
ds@1952
  1328
                    _LexChar = Contents[0];
ds@1952
  1329
                    stripchars = 1;
ds@1952
  1330
                }
ds@1952
  1331
            }
ds@1952
  1332
ds@1952
  1333
            /* Strip N chars */
ds@1952
  1334
            if (stripchars > 0)
ds@1952
  1335
            {
ds@1952
  1336
                Contents.Remove(0, stripchars);
ds@1952
  1337
            }
ds@1952
  1338
ds@1952
  1339
            return _LastToken;
ds@1952
  1340
        }
ds@1952
  1341
    }
ds@1952
  1342
}