sourcemod-central: tools/incparser/IncParser.cs@37373aea4dfe

     1 using System;

     2 using System.Collections.Generic;

     3 using System.IO;

     4 using System.Text;

     6 namespace incparser

7 {

     8     enum LexToken

     9     {

    10         TOKEN_NONE,

    11         TOKEN_INCLUDE,

    12         TOKEN_DEFINE,

    13         TOKEN_CONST,

    14         TOKEN_STRUCT,

    15         TOKEN_FUNCTAG,

    16         TOKEN_FUNCENUM,

    17         TOKEN_ENUM,

    18         TOKEN_NATIVE,

    19         TOKEN_STOCK,

    20         TOKEN_PUBLIC,

    21         TOKEN_FORWARD,

    22         TOKEN_QUOTCHAR,

    23         TOKEN_OPERATOR,

    24         TOKEN_CHARACTER,

    25         /*TOKEN_STRING,*/

    26         TOKEN_LABEL,

    27         TOKEN_SYMBOL,

    28         TOKEN_DOCBLOCK,

    29         TOKEN_EOF,

    30     }

    32     class ParseException : System.Exception

    33     {

    34         public ParseException(string message) : base(message)

    35         {

    36         }

    37     };

    39     class KeywordToken

    40     {

    41         public KeywordToken(string k, LexToken t)

    42         {

    43             keyword = k;

    44             token = t;

    45         }

    46         public string keyword;

    47         public LexToken token;

    48     };

    50     class Tokenizer

    51     {

    52         private static bool s_initialized = false;

    53         private static int s_position = 0;

    54         private static KeywordToken [] s_tokens = null;

    55         public static KeywordToken[] Tokens = null;

    57         public static void Initialize()

    58         {

    59             if (s_initialized)

    60             {

    61                 return;

    62             }

    63             s_tokens = new KeywordToken[12];

    64             s_initialized = true;

    65             s_position = 0;

    67             AddToken("#include", LexToken.TOKEN_INCLUDE);

    68             AddToken("#define", LexToken.TOKEN_DEFINE);

    69             AddToken("native", LexToken.TOKEN_NATIVE);

    70             AddToken("functag", LexToken.TOKEN_FUNCTAG);

    71             AddToken("funcenum", LexToken.TOKEN_FUNCENUM);

    72             AddToken("enum", LexToken.TOKEN_ENUM);

    73             AddToken("stock", LexToken.TOKEN_STOCK);

    74             AddToken("public", LexToken.TOKEN_PUBLIC);

    75             AddToken("forward", LexToken.TOKEN_FORWARD);

    76             AddToken("const", LexToken.TOKEN_CONST);

    77             AddToken("struct", LexToken.TOKEN_STRUCT);

    78             AddToken("operator", LexToken.TOKEN_OPERATOR);

    80             Tokens = s_tokens;

    81         }

    83         private static void AddToken(string lex, LexToken tok)

    84         {

    85             s_tokens[s_position] = new KeywordToken(lex, tok);

    86             s_position++;

    87         }

    88     };

    90     class IncParser

    91     {

    92         private string FileName;

    93         private StringBuilder Contents;

    94         private bool LexPushed;

    95         private LexToken _LastToken;

    96         private string _LexString;

    97         private char _LexChar;

    98         private uint LineNo;

   100         public string LexString

   101         {

   102             get

   103             {

   104                 return _LexString;

   105             }

   106         }

   108         public char LexChar

   109         {

   110             get

   111             {

   112                 return _LexChar;

   113             }

   114         }

   116         public uint GetLineNumber()

   117         {

   118             return LineNo;

   119         }

   121         public IncParser(string file)

   122         {

   123             FileName = file;

   125             /* Clear out lexer stuff */

   126             LexPushed = false;

   127             _LastToken = LexToken.TOKEN_NONE;

   128             _LexString = null;

   129             _LexChar = '\0';

   130             LineNo = 1;

   132             /* Initialize; this can throw an exception */

   133             Initialize();

   134         }

   136         public void Parse(ParseWriter w)

   137         {

   138             w.BeginSection(System.IO.Path.GetFileName(FileName));

   139             LexToken tok = LexToken.TOKEN_NONE;

   140             while ((tok = lex()) != LexToken.TOKEN_EOF)

   141             {

   142                 switch (tok)

   143                 {

   144                     case LexToken.TOKEN_DOCBLOCK:

   145                         {

   146                             w.WritePair("doc", LexString);

   147                             break;

   148                         }

   149                     case LexToken.TOKEN_DEFINE:

   150                         {

   151                             PARSE_Define(w);

   152                             break;

   153                         }

   154                     case LexToken.TOKEN_ENUM:

   155                         {

   156                             PARSE_Enum(w);

   157                             break;

   158                         }

   159                     case LexToken.TOKEN_FORWARD:

   160                         {

   161                             PARSE_Function(tok, w);

   162                             break;

   163                         }

   164                     case LexToken.TOKEN_NATIVE:

   165                         {

   166                             PARSE_Function(tok, w);

   167                             break;

   168                         }

   169                     case LexToken.TOKEN_STOCK:

   170                         {

   171                             PARSE_Function(tok, w);

   172                             break;

   173                         }

   174                     case LexToken.TOKEN_FUNCTAG:

   175                         {

   176                             PARSE_FuncTag(w);

   177                             break;

   178                         }

   179                     case LexToken.TOKEN_FUNCENUM:

   180                         {

   181                             PARSE_FuncEnum(w);

   182                             break;

   183                         }

   184                     case LexToken.TOKEN_INCLUDE:

   185                         {

   186                             PARSE_Include(w);

   187                             break;

   188                         }

   189                     case LexToken.TOKEN_STRUCT:

   190                         {

   191                             PARSE_Struct(w);

   192                             break;

   193                         }

   194                     case LexToken.TOKEN_PUBLIC:

   195                         {

   196                             PARSE_Public(w);

   197                             break;

   198                         }

   199                     default:

   200                         {

   201                             throw new ParseException("Unrecognized token: " + tok);

   202                         }

   203                 }

   204             }

   205             w.EndSection();

   206         }

   208         private void PARSE_Public(ParseWriter w)

   209         {

   210             /* Eat up an optional tag */

   211             MatchToken(LexToken.TOKEN_LABEL);

   213             /* Eat up a name */

   214             NeedToken(LexToken.TOKEN_SYMBOL);

   216             if (MatchChar('='))

   217             {

   218                 ignore_block();

   219             }

   221             NeedChar(';');

   222         }

   224         private void PARSE_Struct(ParseWriter w)

   225         {

   226             /* For now, we completely ignore these (they're not written to the output) */

   227             NeedToken(LexToken.TOKEN_SYMBOL);

   228             NeedChar('{');

   230             bool need_closebrace = true;

   231             bool more_entries = true;

   232             do

   233             {

   234                 if (MatchChar('}'))

   235                 {

   236                     need_closebrace = false;

   237                     break;

   238                 }

   239                 MatchToken(LexToken.TOKEN_CONST);

   240                 MatchToken(LexToken.TOKEN_LABEL);

   241                 NeedToken(LexToken.TOKEN_SYMBOL);

   242                 while (MatchChar('['))

   243                 {

   244                     DiscardUntilChar(']');

   245                     NeedChar(']');

   246                 }

   247                 more_entries = MatchChar(',');

   248                 MatchToken(LexToken.TOKEN_DOCBLOCK);

   249                 if (!more_entries)

   250                 {

   251                     break;

   252                 }

   253             } while (true);

   254             if (need_closebrace)

   255             {

   256                 NeedChar('}');

   257             }

   258             NeedChar(';');

   259         }

   261         private void PARSE_Include(ParseWriter w)

   262         {

   263             /* read until the end of the line */

   264             int index = Contents.ToString().IndexOf('\n');

   265             string value;

   266             if (index == -1)

   267             {

   268                 value = Contents.ToString();

   269                 Contents.Remove(0, value.Length);

   270             }

   271             else if (index == 0)

   272             {

   273                 value = "";

   274             }

   275             else

   276             {

   277                 value = Contents.ToString().Substring(0, index);

   278                 Contents.Remove(0, index);

   279             }

   281             /* Strip whitespace */

   282             value = value.Trim('\r', '\t', ' ');

   284             /* Write */

   285             w.BeginSection("include");

   286             w.WritePair("name", value);

   287             w.EndSection();

   288         }

   290         private void PARSE_FuncEnum(ParseWriter w)

   291         {

   292             w.BeginSection("funcenum");

   294             /* Get the functag name */

   295             NeedToken(LexToken.TOKEN_SYMBOL);

   296             w.WritePair("name", LexString);

   298             NeedChar('{');

   300             bool need_closebrace = true;

   301             do

   302             {

   303                 /* Shortcut out? */

   304                 if (MatchChar('}'))

   305                 {

   306                     need_closebrace = false;

   307                     break;

   308                 }

   310                 /* Start function section */

   311                 w.BeginSection("function");

   313                 if (MatchToken(LexToken.TOKEN_DOCBLOCK))

   314                 {

   315                     w.WritePair("doc", LexString);

   316                 }

   318                 /* Get the return tag */

   319                 if (MatchToken(LexToken.TOKEN_LABEL))

   320                 {

   321                     w.WritePair("return", LexString);

   322                 }

   324                 /* Get function type */

   325                 if (MatchToken(LexToken.TOKEN_PUBLIC))

   326                 {

   327                     w.WritePair("type", "public");

   328                 }

   329                 else if (MatchToken(LexToken.TOKEN_STOCK))

   330                 {

   331                     w.WritePair("type", "stock");

   332                 }

   334                 /* Parse the parameters */

   335                 ParseParameters(w);

   337                 /* End the section */

   338                 w.EndSection();

   340                 if (!MatchChar(','))

   341                 {

   342                     break;

   343                 }

   344             } while (true);

   346             if (need_closebrace)

   347             {

   348                 NeedChar('}');

   349             }

   350             NeedChar(';');

   352             w.EndSection();

   353         }

   355         private void PARSE_FuncTag(ParseWriter w)

   356         {

   357             w.BeginSection("functag");

   359             /* Get the functag name */

   360             NeedToken(LexToken.TOKEN_SYMBOL);

   361             w.WritePair("name", LexString);

   363             if (MatchToken(LexToken.TOKEN_LABEL))

   364             {

   365                 w.WritePair("return", LexString);

   366             }

   368             if (MatchToken(LexToken.TOKEN_PUBLIC))

   369             {

   370                 w.WritePair("type", "public");

   371             }

   372             else if (MatchToken(LexToken.TOKEN_STOCK))

   373             {

   374                 w.WritePair("type", "stock");

   375             }

   377             ParseParameters(w);

   378             NeedChar(';');

   380             w.EndSection();

   381         }

   383         private void PARSE_Function(LexToken tok, ParseWriter w)

   384         {

   385             string tag="", name;

   387             /* Get the return value */

   388             if (MatchToken(LexToken.TOKEN_LABEL))

   389             {

   390                 tag = LexString;

   391             }

   393             if (MatchToken(LexToken.TOKEN_OPERATOR))

   394             {

   395                 /* Check if we're some sort of invalid function */

   396                 DiscardUntilCharOrComment('(', false);

   397                 ParseParameters(null);

   398                 if (tok == LexToken.TOKEN_STOCK)

   399                 {

   400                     ignore_block();

   401                 }

   402                 else if (tok == LexToken.TOKEN_NATIVE || tok == LexToken.TOKEN_FORWARD)

   403                 {

   404                     if (MatchChar('='))

   405                     {

   406                         DiscardUntilCharOrComment(';', false);

   407                     }

   408                     NeedChar(';');

   409                 }

   410                 return;

   411             }

   413             /* Get the name */

   414             NeedToken(LexToken.TOKEN_SYMBOL);

   415             name = LexString;

   417             PARSE_Function(tok, w, tag, name);

   418         }

   420         private void PARSE_Function(LexToken tok, ParseWriter w, string tag, string name)

   421         {

   422             if (tok == LexToken.TOKEN_FORWARD)

   423             {

   424                 w.BeginSection("forward");

   425             }

   426             else if (tok == LexToken.TOKEN_NATIVE)

   427             {

   428                 w.BeginSection("native");

   429             }

   430             else if (tok == LexToken.TOKEN_STOCK)

   431             {

   432                 w.BeginSection("stock");

   433             }

   435             w.WritePair("name", name);

   437             if (tag.Length > 0)

   438             {

   439                 w.WritePair("return", tag);

   440             }

   442             ParseParameters(w);

   444             if (tok == LexToken.TOKEN_STOCK)

   445             {

   446                 ignore_block();

   447             }

   448             else

   449             {

   450                 /* Make sure there's a semicolon */

   451                 if (MatchChar('='))

   452                 {

   453                     DiscardUntilCharOrComment(';', false);

   454                 }

   455                 NeedChar(';');

   456             }

   458             w.EndSection();

   459         }

   461         private void ParseParameters(ParseWriter w)

   462         {

   463             NeedChar('(');

   464             string extra;

   465             do

   466             {

   467                 if (MatchChar(')'))

   468                 {

   469                     return;

   470                 }

   472                 /* Start the parameter */

   473                 if (w != null)

   474                 {

   475                     w.BeginSection("parameter");

   476                 }

   478                 /* Check for a 'const' token */

   479                 if (MatchToken(LexToken.TOKEN_CONST) && w != null)

   480                 {

   481                     w.WritePair("const", "true");

   482                 }

   484                 /* Check if it's by reference */

   485                 if (MatchChar('&') && w != null)

   486                 {

   487                     w.WritePair("byref", "true");

   488                 }

   490                 /* Check for a tag */

   491                 if (MatchToken(LexToken.TOKEN_LABEL) && w != null)

   492                 {

   493                     w.WritePair("tag", LexString);

   494                 }

   496                 /* Get the symbol and write it */

   497                 if (MatchString(0, "...") && w != null)

   498                 {

   499                     w.WritePair("name", "...");

   500                 }

   501                 else

   502                 {

   503                     NeedToken(LexToken.TOKEN_SYMBOL);

   504                     if (w != null)

   505                     {

   506                         w.WritePair("name", LexString);

   507                     }

   508                 }

   510                 /* Check if we have a default value yet */

   511                 bool default_value = false;

   512                 if (MatchChar('='))

   513                 {

   514                     default_value = true;

   515                 }

   516                 else if (MatchChar('['))

   517                 {

   518                     extra = "";

   519                     string temp;

   520                     do

   521                     {

   522                         extra += "[";

   523                         temp = ConsumeUntilChar(']');

   524                         if (temp != null)

   525                         {

   526                             extra += temp;

   527                         }

   528                         extra += "]";

   529                         NeedChar(']');

   530                     } while (MatchChar('['));

   531                     w.WritePair("decoration", extra);

   532                 }

   534                 /* If we have a default value, get it */

   535                 if (default_value || MatchChar('='))

   536                 {

   537                     if ((extra = ConsumeParamDefValue()) == null)

   538                     {

   539                         throw new ParseException("Expected a default value; found none");

   540                     }

   541                     if (w != null)

   542                     {

   543                         w.WritePair("defval", extra);

   544                     }

   545                 }

   547                 /* End the parameter */

   548                 if (w != null)

   549                 {

   550                     w.EndSection();

   551                 }

   553                 if (!MatchChar(','))

   554                 {

   555                     break;

   556                 }

   557             } while (true);

   559             NeedChar(')');

   560         }

   562         private void PARSE_Enum(ParseWriter w)

   563         {

   564             string name = "";

   565             if (MatchToken(LexToken.TOKEN_SYMBOL))

   566             {

   567                 name = LexString;

   568             }

   570             w.BeginSection("enum");

   571             w.WritePair("name", name);

   572             w.BeginSection("properties");

   574             NeedChar('{');

   575             bool more_entries = true;

   576             bool need_closebrace = true;

   577             do

   578             {

   579                 if (MatchChar('}'))

   580                 {

   581                     need_closebrace = false;

   582                     break;

   583                 }

   584                 NeedToken(LexToken.TOKEN_SYMBOL);

   585                 name = LexString;

   586                 if (MatchChar('='))

   587                 {

   588                     DiscardUntilCharOrComment(',', true);

   589                 }

   590                 more_entries = MatchChar(',');

   591                 w.WritePair("name", name);

   592                 if (MatchToken(LexToken.TOKEN_DOCBLOCK))

   593                 {

   594                     w.WritePair("doc", LexString);

   595                 }

   596             } while (more_entries);

   597             if (need_closebrace)

   598             {

   599                 NeedChar('}');

   600             }

   601             NeedChar(';');

   602             w.EndSection();

   603             w.EndSection();

   604         }

   606         private void PARSE_Define(ParseWriter w)

   607         {

   608             /* first, get the symbol name */

   609             NeedToken(LexToken.TOKEN_SYMBOL);

   610             string name = LexString;

   611             string value;

   613             /* read until we hit the end of a line OR a comment */

   614             int endpoint = -1;

   615             for (int i = 0; i < Contents.Length; i++)

   616             {

   617                 if (Contents[i] == '\n')

   618                 {

   619                     endpoint = i;

   620                     break;

   621                 }

   622                 else if (CheckString(i, "/**"))

   623                 {

   624                     endpoint = i;

   625                     break;

   626                 }

   627             }

   629             if (endpoint < 1)

   630             {

   631                 value = "";

   632             }

   633             else

   634             {

   635                 value = Contents.ToString().Substring(0, endpoint);

   636                 Contents.Remove(0, endpoint);

   637             }

   639             /* Write */

   640             w.BeginSection("define");

   641             w.WritePair("name", name);

   642             w.WritePair("linetext", value);

   643             w.EndSection();

   644         }

   646         /* Reads up to a character and discards all text before it.

   647          */

   648         private void DiscardUntilChar(char c)

   649         {

   650             int eol = Contents.ToString().IndexOf(c);

   652             if (eol < 1)

   653             {

   654                 return;

   655             }

   657             Contents.Remove(0, eol);

   658         }

   660         /* Same as DiscardUntilChar, except the discarded

   661          * text is returned first.

   662          */

   663         private string ConsumeUntilChar(char c)

   664         {

   665             int eol = Contents.ToString().IndexOf(c);

   667             if (eol < 1)

   668             {

   669                 return null;

   670             }

   672             string rest = Contents.ToString().Substring(0, eol);

   673             Contents.Remove(0, eol);

   674             return rest;

   675         }

   677         /**

   678          * Returns a parameter's default value.

   679          */

   680         private string ConsumeParamDefValue()

   681         {

   682             int eol = FindEndOfExprLine(0);

   683             if (eol < 1)

   684             {

   685                 return null;

   686             }

   688             string rest = Contents.ToString().Substring(0, eol);

   689             Contents.Remove(0, eol);

   690             return rest;

   691         }

   693         /**

   694          * Discards all data up to a certain character, ignoring

   695          * the contents, unless specified to stop at a doc block comment.

   696          */

   697         private void DiscardUntilCharOrComment(char c, bool stop_at_comment)

   698         {

   699             for (int i = 0; i < Contents.Length; i++)

   700             {

   701                 if (Contents[i] == '\n')

   702                 {

   703                     LineNo++;

   704                 }

   705                 else if (Contents[i] == c)

   706                 {

   707                     if (i > 0)

   708                     {

   709                         Contents.Remove(0, i);

   710                     }

   711                     return;

   712                 }

   713                 else if (stop_at_comment && CheckString(i, "/**"))

   714                 {

   715                     if (i > 0)

   716                     {

   717                         Contents.Remove(0, i);

   718                     }

   719                     return;

   720                 }

   721             }

   722         }

   724         /**

   725          * Finds the end of an expression

   726          */

   727         private int FindEndOfExprLine(int start)

   728         {

   729             for (int i = start; i < Contents.Length; i++)

   730             {

   731                 if (Contents[i] == '{')

   732                 {

   733                     if ((i = FindEndOfConsumptionLine('}', start+1)) == -1)

   734                     {

   735                         throw new ParseException("Expected token: } (found none)");

   736                     }

   737                 }

   738                 else if (Contents[i] == ')' || Contents[i] == ',')

   739                 {

   740                     return i;

   741                 }

   742             }

   744             return -1;

   745         }

   747         private int FindEndOfConsumptionLine(char c, int start)

   748         {

   749             for (int i = start; i < Contents.Length; i++)

   750             {

   751                 if (Contents[i] == '{')

   752                 {

   753                     if ((i = FindEndOfConsumptionLine('}', i + 1)) == -1)

   754                     {

   755                         throw new ParseException("Expected token: } (found none)");

   756                     }

   757                 }

   758                 else if (Contents[i] == c)

   759                 {

   760                     return i;

   761                 }

   762             }

   764             return -1;

   765         }

   767         private void Initialize()

   768         {

   769             /* Open file, read it, and close it */

   770             StreamReader sr = File.OpenText(FileName);

   771             Contents = new StringBuilder(sr.ReadToEnd());

   772             sr.Close();

   774             /* Strip comments (may throw an exception) */

   775             preprocess();

   777             /* Lastly, initialize the tokenizer */

   778             Tokenizer.Initialize();

   779         }

   781         /* Consumes a token iff it matches */

   782         private bool MatchToken(LexToken token)

   783         {

   784             LexToken tok = lex();

   785             if (tok != token)

   786             {

   787                 lexpush();

   788                 return false;

   789             }

   790             return true;

   791         }

   793         /* Consumes a token and aborts the parser if there is no match */

   794         private void NeedToken(LexToken token)

   795         {

   796             if (!MatchToken(token))

   797             {

   798                 throw new ParseException("Expected token: " + token);

   799             }

   800         }

   802         /* Consumes a character iff it matches */

   803         private bool MatchChar(char c)

   804         {

   805             if (lex() != LexToken.TOKEN_CHARACTER)

   806             {

   807                 lexpush();

   808                 return false;

   809             }

   810             if (_LexChar != c)

   811             {

   812                 lexpush();

   813                 return false;

   814             }

   815             return true;

   816         }

   818         /* Consumes a character or aborts the parser if there is no match */

   819         private void NeedChar(char c)

   820         {

   821             if (!MatchChar(c))

   822             {

   823                 throw new ParseException("Expected character: " + c);

   824             }

   825         }

   827         /* Saves the lexer state so we don't reparse unnecessarily */

   828         private void lexpush()

   829         {

   830             if (LexPushed)

   831             {

   832                 throw new ParseException("Lexer cannot be pushed twice");

   833             }

   834             LexPushed = true;

   835         }

   837         /* Attempts to match a string against the next stream tokens.

   838          * On success, the token stream is advanced.

   839          */

   840         private bool MatchString(int startPos, string str)

   841         {

   842             if (str.Length > Contents.Length - startPos)

   843             {

   844                 return false;

   845             }

   846             for (int i = 0; i < str.Length; i++)

   847             {

   848                 if (str[i] != Contents[startPos + i])

   849                 {

   850                     return false;

   851                 }

   852             }

   853             Contents.Remove(startPos, str.Length);

   854             return true;

   855         }

   857         /* Attempts to match a string against the next stream tokens */

   858         private bool CheckString(int startPos, string str)

   859         {

   860             if (str.Length > Contents.Length - startPos)

   861             {

   862                 return false;

   863             }

   864             for (int i = 0; i < str.Length; i++)

   865             {

   866                 if (str[i] != Contents[startPos + i])

   867                 {

   868                     return false;

   869                 }

   870             }

   871             return true;

   872         }

   874         /* Ignores a block starting and ending with { and } respectively */

   875         private void ignore_block()

   876         {

   877             /* Hrm, this is tricky.  First, expect an open brace. */

   878             NeedChar('{');

   879             /* Consume the entire function by keeping track of braces */

   880             int numBraces = 1;

   881             bool in_string = false;

   882             bool found_end = false;

   883             for (int i = 0; i < Contents.Length; i++)

   884             {

   885                 if (Contents[i] == '\n')

   886                 {

   887                     LineNo++;

   888                 }

   889                 if (!in_string)

   890                 {

   891                     if (Contents[i] == '"')

   892                     {

   893                         in_string = true;

   894                     }

   895                     else if (Contents[i] == '{')

   896                     {

   897                         numBraces++;

   898                     }

   899                     else if (Contents[i] == '}')

   900                     {

   901                         if (numBraces == 0)

   902                         {

   903                             throw new ParseException("Unable to find a matching open brace");

   904                         }

   905                         if (--numBraces == 0)

   906                         {

   907                             Contents.Remove(0, i + 1);

   908                             found_end = true;

   909                             break;

   910                         }

   911                     }

   912                 }

   913                 else

   914                 {

   915                     /* It is possible to be in a string and at the first character */

   916                     if (Contents[i] == '"' && Contents[i - 1] != '\\')

   917                     {

   918                         in_string = false;

   919                     }

   920                 }

   921             }

   922             if (!found_end)

   923             {

   924                 throw new ParseException("Unable to find a matching close brace");

   925             }

   926         }

   928         /* Strips all non-doc C and C++ style comments. as well as unused pragmas */

   929         private void preprocess()

   930         {

   931             if (Contents.Length == 1)

   932             {

   933                 return;

   934             }

   936             bool in_comment = false;

   937             bool is_ml = false;

   938             bool is_doc = false;

   939             bool is_newline = true;

   941             for (int i=0; i<Contents.Length-1; i++)

   942             {

   943                 if (Contents[i] == '\n')

   944                 {

   945                     is_newline = true;

   946                 }

   947                 if (!in_comment)

   948                 {

   949                     if (Contents[i] == '/')

   950                     {

   951                         if (Contents[i+1] == '/')

   952                         {

   953                             in_comment = true;

   954                             is_ml = false;

   955                             is_doc = false;

   956                         } else if (Contents[i+1] == '*') {

   957                             /* Detect whether this is a doc comment or not */

   958                             if (Contents.Length - i >= 3

   959                                 && Contents[i+2] == '*')

   960                             {

   961                                 is_doc = true;

   962                             } else {

   963                                 is_doc = false;

   964                             }

   965                             in_comment = true;

   966                             is_ml = true;

   967                         }

   968                         if (in_comment && !is_doc)

   969                         {

   970                             Contents[i] = ' ';

   971                             Contents[i+1] = ' ';        /* Just for safety */

   972                         }

   973                     }

   974                     else if (is_newline)

   975                     {

   976                         if (Contents[i] == '#')

   977                         {

   978                             if (!CheckString(i, "#define")

   979                                 && !CheckString(i, "#include"))

   980                             {

   981                                 /* Ignore! */

   982                                 Contents[i] = ' ';

   983                                 in_comment = true;

   984                                 is_doc = false;

   985                                 is_ml = false;

   986                             }

   987                             is_newline = false;

   988                         }

   989                         else if (!Char.IsWhiteSpace(Contents[i]))

   990                         {

   991                             is_newline = false;

   992                         }

   993                     }

   994                 } else {

   995                     if (is_ml)

   996                     {

   997                         if (Contents[i] == '*' && Contents[i+1] == '/')

   998                         {

   999                             if (!is_doc)

  1000                             {

  1001                                 Contents[i] = ' ';

  1002                                 Contents[i+1] = ' ';

  1003                             }

  1004                             in_comment = false;

  1005                             i++;    /* Don't look at the next character */

  1006                         }

  1007                         else if (!is_doc && Contents[i] != '\n')

  1008                         {

  1009                             /* Erase comment */

  1010                             Contents[i] = ' ';

  1011                         }

  1012                     } else {

  1013                         if (is_newline)

  1014                         {

  1015                             in_comment = false;

  1016                         }

  1017                         else

  1018                         {

  1019                             /* Erase comment */

  1020                             Contents[i] = ' ';

  1021                         }

  1022                     }

  1023                 }

  1024             }

  1026             if (in_comment)

  1027             {

  1028                 throw new ParseException("Multi-line comment has no terminator");

  1029             }

  1030         }

  1032         /* Strips whitespace */

  1033         private void trim_whitespace_pre()

  1034         {

  1035             int spaces = 0;

  1036             while (spaces < Contents.Length

  1037                    && Contents[spaces] != '\0'

  1038                    && Char.IsWhiteSpace(Contents[spaces]))

  1039             {

  1040                 if (Contents[spaces] == '\n')

  1041                 {

  1042                     LineNo++;

  1043                 }

  1044                 spaces++;

  1045             }

  1046             if (spaces > 0)

  1047             {

  1048                 Contents.Remove(0, spaces);

  1049             }

  1050         }

  1052         private void preprocess_line()

  1053         {

  1054             trim_whitespace_pre();

  1055         }

  1057         private bool IsSymbolChar(char c, bool first)

  1058         {

  1059             bool valid = (first ? Char.IsLetter(c) : Char.IsLetterOrDigit(c));

  1060             return (valid || (c == '_'));

  1061         }

  1063         private bool IsEscapedChar(char c)

  1064         {

  1065             return (c == 'n'

  1066                     || c == 'r'

  1067                     || c == 't'

  1068                     || c == 'v'

  1069                     || c == '\\'

  1070                     || c == '"');

  1071         }

  1073         private bool IsNotKeywordChar(char c)

  1074         {

  1075             return (Char.IsWhiteSpace(c)

  1076                     || (c == '-' || c == '+')

  1077                     || (c == '/' || c == '*')

  1078                     || (c == '(' || c == ')')

  1079                     || (c == '=' || c == '%')

  1080                     || (c == '>' || c == '<')

  1081                     || (c == '!' || c == '~')

  1082                     );

  1083         }

  1085         /* Consumes one lexical token and gathers information about it */

  1086         private LexToken lex()

  1087         {

  1088             if (LexPushed)

  1089             {

  1090                 LexPushed = false;

  1091                 return _LastToken;

  1092             }

  1094             /* Number of chars we will be deleting from the input */

  1095             int stripchars = 0;

  1097             /* Clear our state */

  1098             _LastToken = LexToken.TOKEN_NONE;

  1100             /* Remove stuff we don't want from the line */

  1101             preprocess_line();

  1103             if (Contents.Length < 1)

  1104             {

  1105                 _LastToken = LexToken.TOKEN_EOF;

  1106                 return _LastToken;

  1107             }

  1109             /* Get the token list */

  1110             KeywordToken[] tokens = Tokenizer.Tokens;

  1111             for (int i = 0; i < tokens.Length; i++)

  1112             {

  1113                 if (!CheckString(0, tokens[i].keyword))

  1114                 {

  1115                     continue;

  1116                 }

  1117                 int len = tokens[i].keyword.Length;

  1118                 /* Now check to see what the next token is */

  1119                 if (Contents.Length == len

  1120                     || IsNotKeywordChar(Contents[len]))

  1121                 {

  1122                     /* We have a token match! */

  1123                     _LastToken = tokens[i].token;

  1124                     _LexString = null;

  1125                     _LexChar = '\0';

  1126                     stripchars = len;

  1127                     break;

  1128                 }

  1129             }

  1131             if (_LastToken == LexToken.TOKEN_NONE)

  1132             {

  1133                 /* See if we can try to read a symbol */

  1134                 if (IsSymbolChar(Contents[0], true))

  1135                 {

  1136                     int characters = 1;

  1137                     while (Contents[characters] != '\0'

  1138                            && IsSymbolChar(Contents[characters], false))

  1139                     {

  1140                         characters++;

  1141                     }

  1142                     stripchars = characters;

  1143                     /* We're done! See what's next.. */

  1144                     if (Contents[characters] == ':')

  1145                     {

  1146                         _LastToken = LexToken.TOKEN_LABEL;

  1147                         stripchars++;

  1148                     }

  1149                     else

  1150                     {

  1151                         _LastToken = LexToken.TOKEN_SYMBOL;

  1152                     }

  1153                     _LexString = Contents.ToString().Substring(0, characters);

  1154                     _LexChar = _LexString[0];

  1155                 }/*

  1156                 else if (Contents[0] == '"')

  1157                 {

  1158                     bool reached_end = false;

  1159                     StringBuilder builder = new StringBuilder();

  1160                     for (int i = 1; i < Contents.Length; i++)

  1161                     {

  1162                         if (i < Contents.Length - 1

  1163                             && Contents[i] == '\\'

  1164                             && IsEscapedChar(Contents[i+1]))

  1165                         {

  1166                             i++;

  1167                         }

  1168                         else if (Contents[i] == '"')

  1169                         {

  1170                             reached_end = true;

  1171                             break;

  1172                         }

  1173                         builder.Append(Contents[i]);

  1174                     }

  1175                     if (!reached_end)

  1176                     {

  1177                         throw new ParseException("Expected end of string; none found");

  1178                     }

  1179                     _LexString = builder.ToString();

  1180                     _LexChar = _LexString.Length > 0 ? _LexString[0] : '\0';

  1181                 }*/

  1182                 else if (Contents[0] == '\'')

  1183                 {

  1184                     char c = '\0';

  1185                     if (Contents.Length < 3)

  1186                     {

  1187                         throw new ParseException("Expected end of character; none found");

  1188                     }

  1189                     else if (Contents[1] == '\\')

  1190                     {

  1191                         if (Contents.Length < 4 || !IsEscapedChar(Contents[2]))

  1192                         {

  1193                             throw new ParseException("Expected end of character; none found");

  1194                         }

  1195                         if (Contents[2] == 'r')

  1196                         {

  1197                             c = '\r';

  1198                         }

  1199                         else if (Contents[2] == 'n')

  1200                         {

  1201                             c = '\n';

  1202                         }

  1203                         else if (Contents[2] == 't')

  1204                         {

  1205                             c = '\t';

  1206                         }

  1207                         else if (Contents[2] == 'v')

  1208                         {

  1209                             c = '\v';

  1210                         }

  1211                         else if (Contents[2] == '\\')

  1212                         {

  1213                             c = '\\';

  1214                         }

  1215                         else if (Contents[2] == '"')

  1216                         {

  1217                             c = '"';

  1218                         }

  1219                         stripchars = 4;

  1220                     }

  1221                     else

  1222                     {

  1223                         c = Contents[1];

  1224                         stripchars = 3;

  1225                     }

  1226                     _LexString = c.ToString();

  1227                     _LexChar = c;

  1228                     _LastToken = LexToken.TOKEN_QUOTCHAR;

  1229                 }

  1230                 else if (CheckString(0, "/**"))

  1231                 {

  1232                     int endpoint = 0;

  1233                     for (int i=3; i<Contents.Length-1; i++)

  1234                     {

  1235                         if (Contents[i] == '\n')

  1236                         {

  1237                             LineNo++;

  1238                         }

  1239                         else if (Contents[i] == '*' && Contents[i+1] == '/')

  1240                         {

  1241                             endpoint = i+1;

  1242                             break;

  1243                         }

  1244                     }

  1245                     if (endpoint == 0)

  1246                     {

  1247                         throw new ParseException("Expected end of multi-line comment; found end of file");

  1248                     }

  1249                     /* We have to factor one more character in because we're zero based */

  1250                     endpoint++;

  1251                     _LastToken = LexToken.TOKEN_DOCBLOCK;

  1252                     _LexString = Contents.ToString().Substring(0, endpoint);

  1253                     _LexChar = _LexString[0];

  1254                     stripchars = endpoint;

  1255                 }

  1256                 else

  1257                 {

  1258                     _LastToken = LexToken.TOKEN_CHARACTER;

  1259                     _LexString = Contents[0].ToString();

  1260                     _LexChar = Contents[0];

  1261                     stripchars = 1;

  1262                 }

  1263             }

  1265             /* Strip N chars */

  1266             if (stripchars > 0)

  1267             {

  1268                 Contents.Remove(0, stripchars);

  1269             }

  1271             return _LastToken;

  1272         }

  1273     }

  1274 }

author	David Anderson <dvander@alliedmods.net>
	Sun Oct 28 08:24:47 2007 +0000 (2007-10-28)
changeset 1649	37373aea4dfe
parent 824	0e9a2c98c9aa
child 2019	9f473114a511
permissions	-rw-r--r--