sourcemod-central: tools/incparser/IncParser.cs@4e43bcc83a54

     1 using System;

     2 using System.Collections.Generic;

     3 using System.IO;

     4 using System.Text;

     6 namespace incparser

7 {

     8     enum LexToken

     9     {

    10         TOKEN_NONE,

    11         TOKEN_INCLUDE,

    12         TOKEN_DEFINE,

    13         TOKEN_CONST,

    14         TOKEN_STRUCT,

    15         TOKEN_FUNCTAG,

    16         TOKEN_FUNCENUM,

    17         TOKEN_ENUM,

    18         TOKEN_NATIVE,

    19         TOKEN_STOCK,

    20         TOKEN_PUBLIC,

    21         TOKEN_FORWARD,

    22         TOKEN_QUOTCHAR,

    23         TOKEN_OPERATOR,

    24         TOKEN_CHARACTER,

    25         /*TOKEN_STRING,*/

    26         TOKEN_LABEL,

    27         TOKEN_SYMBOL,

    28         TOKEN_DOCBLOCK,

    29         TOKEN_EOF,

    30     }

    32     class ParseException : System.Exception

    33     {

    34         public ParseException(string message) : base(message)

    35         {

    36         }

    37     };

    39     class KeywordToken

    40     {

    41         public KeywordToken(string k, LexToken t)

    42         {

    43             keyword = k;

    44             token = t;

    45         }

    46         public string keyword;

    47         public LexToken token;

    48     };

    50     class Tokenizer

    51     {

    52         private static bool s_initialized = false;

    53         private static int s_position = 0;

    54         private static KeywordToken [] s_tokens = null;

    55         public static KeywordToken[] Tokens = null;

    57         public static void Initialize()

    58         {

    59             if (s_initialized)

    60             {

    61                 return;

    62             }

    63             s_tokens = new KeywordToken[12];

    64             s_initialized = true;

    65             s_position = 0;

    67             AddToken("#include", LexToken.TOKEN_INCLUDE);

    68             AddToken("#define", LexToken.TOKEN_DEFINE);

    69             AddToken("native", LexToken.TOKEN_NATIVE);

    70             AddToken("functag", LexToken.TOKEN_FUNCTAG);

    71             AddToken("funcenum", LexToken.TOKEN_FUNCENUM);

    72             AddToken("enum", LexToken.TOKEN_ENUM);

    73             AddToken("stock", LexToken.TOKEN_STOCK);

    74             AddToken("public", LexToken.TOKEN_PUBLIC);

    75             AddToken("forward", LexToken.TOKEN_FORWARD);

    76             AddToken("const", LexToken.TOKEN_CONST);

    77             AddToken("struct", LexToken.TOKEN_STRUCT);

    78             AddToken("operator", LexToken.TOKEN_OPERATOR);

    80             Tokens = s_tokens;

    81         }

    83         private static void AddToken(string lex, LexToken tok)

    84         {

    85             s_tokens[s_position] = new KeywordToken(lex, tok);

    86             s_position++;

    87         }

    88     };

    90     class IncParser

    91     {

    92         private string FileName;

    93         private StringBuilder Contents;

    94         private bool LexPushed;

    95         private LexToken _LastToken;

    96         private string _LexString;

    97         private char _LexChar;

    98         private uint LineNo;

   100         public string LexString

   101         {

   102             get

   103             {

   104                 return _LexString;

   105             }

   106         }

   108         public char LexChar

   109         {

   110             get

   111             {

   112                 return _LexChar;

   113             }

   114         }

   116         public uint GetLineNumber()

   117         {

   118             return LineNo;

   119         }

   121         public IncParser(string file)

   122         {

   123             FileName = file;

   125             /* Clear out lexer stuff */

   126             LexPushed = false;

   127             _LastToken = LexToken.TOKEN_NONE;

   128             _LexString = null;

   129             _LexChar = '\0';

   130             LineNo = 1;

   132             /* Initialize; this can throw an exception */

   133             Initialize();

   134         }

   136          public void Parse(ParseWriter w)

   137         {

   138             w.BeginSection(System.IO.Path.GetFileName(FileName));

   139             LexToken tok = LexToken.TOKEN_NONE;

   140             while ((tok = lex()) != LexToken.TOKEN_EOF)

   141             {

   142                 switch (tok)

   143                 {

   144                     case LexToken.TOKEN_DOCBLOCK:

   145                         {

   146                             w.WritePair("doc", LexString);

   147                             break;

   148                         }

   149                     case LexToken.TOKEN_DEFINE:

   150                         {

   151                             PARSE_Define(w);

   152                             break;

   153                         }

   154                     case LexToken.TOKEN_ENUM:

   155                         {

   156                             PARSE_Enum(w);

   157                             break;

   158                         }

   159                     case LexToken.TOKEN_FORWARD:

   160                         {

   161                             PARSE_Function(tok, w);

   162                             break;

   163                         }

   164                     case LexToken.TOKEN_NATIVE:

   165                         {

   166                             PARSE_Function(tok, w);

   167                             break;

   168                         }

   169                     case LexToken.TOKEN_STOCK:

   170                         {

   171                             PARSE_Function(tok, w);

   172                             break;

   173                         }

   174                     case LexToken.TOKEN_FUNCTAG:

   175                         {

   176                             PARSE_FuncTag(w);

   177                             break;

   178                         }

   179                     case LexToken.TOKEN_FUNCENUM:

   180                         {

   181                             PARSE_FuncEnum(w);

   182                             break;

   183                         }

   184                     case LexToken.TOKEN_INCLUDE:

   185                         {

   186                             PARSE_Include(w);

   187                             break;

   188                         }

   189                     case LexToken.TOKEN_STRUCT:

   190                         {

   191                             PARSE_Struct(w);

   192                             break;

   193                         }

   194                     case LexToken.TOKEN_PUBLIC:

   195                         {

   196                             PARSE_Public(w);

   197                             break;

   198                         }

   199                     default:

   200                         {

   201                             if (_LexString == "static" || _LexString == "new")

   202                             {

   203                                 PARSE_Public(w);

   204                                 break;

   205                             }

   206                             throw new ParseException("Unrecognized token: " + tok + " " + _LexString);

   207                         }

   208                 }

   209             }

   210             w.EndSection();

   211         }

   213         private void PARSE_Public(ParseWriter w)

   214         {

   215             MatchToken(LexToken.TOKEN_CONST);

   217             /* Eat up an optional tag */

   218             MatchToken(LexToken.TOKEN_LABEL);

   220             /* Eat up a name */

   221             NeedToken(LexToken.TOKEN_SYMBOL);

   223             while (MatchChar('['))

   224             {

   225                 DiscardUntilChar(']');

   226                 NeedChar(']');

   227             }

   229             if (MatchChar('='))

   230             {

   231                 ignore_block();

   232             }

   233             else if (MatchChar('(') && MatchChar(')'))

   234             {

   235                 ignore_block();

   236                 return;

   237             }

   239             NeedChar(';');

   240         }

   242         private void PARSE_Struct(ParseWriter w)

   243         {

   244             /* For now, we completely ignore these (they're not written to the output) */

   245             NeedToken(LexToken.TOKEN_SYMBOL);

   246             w.structList.Add(LexString);

   247             NeedChar('{');

   249             bool need_closebrace = true;

   250             bool more_entries = true;

   251             do

   252             {

   253                 if (MatchChar('}'))

   254                 {

   255                     need_closebrace = false;

   256                     break;

   257                 }

   258                 MatchToken(LexToken.TOKEN_CONST);

   259                 MatchToken(LexToken.TOKEN_LABEL);

   260                 NeedToken(LexToken.TOKEN_SYMBOL);

   261                 while (MatchChar('['))

   262                 {

   263                     DiscardUntilChar(']');

   264                     NeedChar(']');

   265                 }

   266                 more_entries = MatchChar(',');

   267                 MatchToken(LexToken.TOKEN_DOCBLOCK);

   268                 if (!more_entries)

   269                 {

   270                     break;

   271                 }

   272             } while (true);

   273             if (need_closebrace)

   274             {

   275                 NeedChar('}');

   276             }

   277             NeedChar(';');

   278         }

   280         private void PARSE_Include(ParseWriter w)

   281         {

   282             /* read until the end of the line */

   283             int index = Contents.ToString().IndexOf('\n');

   284             string value;

   285             if (index == -1)

   286             {

   287                 value = Contents.ToString();

   288                 Contents.Remove(0, value.Length);

   289             }

   290             else if (index == 0)

   291             {

   292                 value = "";

   293             }

   294             else

   295             {

   296                 value = Contents.ToString().Substring(0, index);

   297                 Contents.Remove(0, index);

   298             }

   300             /* Strip whitespace */

   301             value = value.Trim('\r', '\t', ' ');

   303             /* Write */

   304             w.BeginSection("include");

   305             w.WritePair("name", value);

   306             w.EndSection();

   307         }

   309         private void PARSE_FuncEnum(ParseWriter w)

   310         {

   311             w.BeginSection("funcenum");

   313             /* Get the functag name */

   314             NeedToken(LexToken.TOKEN_SYMBOL);

   315             w.WritePair("name", LexString);

   316             w.funcenumList.Add(LexString);

   318             NeedChar('{');

   320             bool need_closebrace = true;

   321             do

   322             {

   323                 /* Shortcut out? */

   324                 if (MatchChar('}'))

   325                 {

   326                     need_closebrace = false;

   327                     break;

   328                 }

   330                 /* Start function section */

   331                 w.BeginSection("function");

   333                 if (MatchToken(LexToken.TOKEN_DOCBLOCK))

   334                 {

   335                     w.WritePair("doc", LexString);

   336                 }

   338                 /* Get the return tag */

   339                 if (MatchToken(LexToken.TOKEN_LABEL))

   340                 {

   341                     w.WritePair("return", LexString);

   342                 }

   344                 /* Get function type */

   345                 if (MatchToken(LexToken.TOKEN_PUBLIC))

   346                 {

   347                     w.WritePair("type", "public");

   348                 }

   349                 else if (MatchToken(LexToken.TOKEN_STOCK))

   350                 {

   351                     w.WritePair("type", "stock");

   352                 }

   354                 /* Parse the parameters */

   355                 ParseParameters(w);

   357                 /* End the section */

   358                 w.EndSection();

   360                 if (!MatchChar(','))

   361                 {

   362                     break;

   363                 }

   364             } while (true);

   366             if (need_closebrace)

   367             {

   368                 NeedChar('}');

   369             }

   370             NeedChar(';');

   372             w.EndSection();

   373         }

   375         private void PARSE_FuncTag(ParseWriter w)

   376         {

   377             w.BeginSection("functag");

   379             if (MatchToken(LexToken.TOKEN_PUBLIC))

   380             {

   381                 w.WritePair("type", "public");

   382             }

   383             else if (MatchToken(LexToken.TOKEN_STOCK))

   384             {

   385                 w.WritePair("type", "stock");

   386             }

   388             if (MatchToken(LexToken.TOKEN_LABEL))

   389             {

   390                 w.WritePair("return", LexString);

   391             }

   393             /* Get the functag name */

   394             NeedToken(LexToken.TOKEN_SYMBOL);

   395             w.WritePair("name", LexString);

   396             w.functagList.Add(LexString);

   398             ParseParameters(w);

   399             NeedChar(';');

   401             w.EndSection();

   402         }

   404         private void PARSE_Function(LexToken tok, ParseWriter w)

   405         {

   406             string tag="", name;

   408             /* Get the return value */

   409             if (MatchToken(LexToken.TOKEN_LABEL))

   410             {

   411                 tag = LexString;

   412             }

   414             if (MatchToken(LexToken.TOKEN_OPERATOR))

   415             {

   416                 /* Check if we're some sort of invalid function */

   417                 DiscardUntilCharOrComment('(', false);

   418                 ParseParameters(null);

   419                 if (tok == LexToken.TOKEN_STOCK)

   420                 {

   421                     ignore_block();

   422                 }

   423                 else if (tok == LexToken.TOKEN_NATIVE || tok == LexToken.TOKEN_FORWARD)

   424                 {

   425                     if (MatchChar('='))

   426                     {

   427                         DiscardUntilCharOrComment(';', false);

   428                     }

   429                     NeedChar(';');

   430                 }

   431                 return;

   432             }

   434             /* Get the name */

   435             NeedToken(LexToken.TOKEN_SYMBOL);

   436             name = LexString;

   438             PARSE_Function(tok, w, tag, name);

   439         }

   441         private void PARSE_Function(LexToken tok, ParseWriter w, string tag, string name)

   442         {

   443             if (tok == LexToken.TOKEN_FORWARD)

   444             {

   445                 w.BeginSection("forward");

   446                 w.forwardList.Add(name);

   447             }

   448             else if (tok == LexToken.TOKEN_NATIVE)

   449             {

   450                 w.BeginSection("native");

   451                 w.nativeList.Add(name);

   452             }

   453             else if (tok == LexToken.TOKEN_STOCK)

   454             {

   455                 w.BeginSection("stock");

   456                 w.stockList.Add(name);

   457             }

   459             w.WritePair("name", name);

   461             if (tag.Length > 0)

   462             {

   463                 w.WritePair("return", tag);

   464             }

   466             ParseParameters(w);

   468             if (tok == LexToken.TOKEN_STOCK)

   469             {

   470                 ignore_block();

   471             }

   472             else

   473             {

   474                 /* Make sure there's a semicolon */

   475                 if (MatchChar('='))

   476                 {

   477                     DiscardUntilCharOrComment(';', false);

   478                 }

   479                 NeedChar(';');

   480             }

   482             w.EndSection();

   483         }

   485         private void ParseParameters(ParseWriter w)

   486         {

   487             NeedChar('(');

   488             string extra;

   489             do

   490             {

   491                 if (MatchChar(')'))

   492                 {

   493                     return;

   494                 }

   496                 /* Start the parameter */

   497                 if (w != null)

   498                 {

   499                     w.BeginSection("parameter");

   500                 }

   502                 /* Check for a 'const' token */

   503                 if (MatchToken(LexToken.TOKEN_CONST) && w != null)

   504                 {

   505                     w.WritePair("const", "true");

   506                 }

   508                 /* Check if it's by reference */

   509                 if (MatchChar('&') && w != null)

   510                 {

   511                     w.WritePair("byref", "true");

   512                 }

   514                 /* Check for a tag */

   515                 if (MatchToken(LexToken.TOKEN_LABEL) && w != null)

   516                 {

   517                     w.WritePair("tag", LexString);

   518                 }

   520                 /* Get the symbol and write it */

   521                 if (MatchString(0, "...") && w != null)

   522                 {

   523                     w.WritePair("name", "...");

   524                 }

   525                 else

   526                 {

   527                     NeedToken(LexToken.TOKEN_SYMBOL);

   528                     if (w != null)

   529                     {

   530                         w.WritePair("name", LexString);

   531                     }

   532                 }

   534                 /* Check if we have a default value yet */

   535                 bool default_value = false;

   536                 if (MatchChar('='))

   537                 {

   538                     default_value = true;

   539                 }

   540                 else if (MatchChar('['))

   541                 {

   542                     extra = "";

   543                     string temp;

   544                     do

   545                     {

   546                         extra += "[";

   547                         temp = ConsumeUntilChar(']');

   548                         if (temp != null)

   549                         {

   550                             extra += temp;

   551                         }

   552                         extra += "]";

   553                         NeedChar(']');

   554                     } while (MatchChar('['));

   555                     w.WritePair("decoration", extra);

   556                 }

   558                 /* If we have a default value, get it */

   559                 if (default_value || MatchChar('='))

   560                 {

   561                     if ((extra = ConsumeParamDefValue()) == null)

   562                     {

   563                         throw new ParseException("Expected a default value; found none");

   564                     }

   565                     if (w != null)

   566                     {

   567                         w.WritePair("defval", extra);

   568                     }

   569                 }

   571                 /* End the parameter */

   572                 if (w != null)

   573                 {

   574                     w.EndSection();

   575                 }

   577                 if (!MatchChar(','))

   578                 {

   579                     break;

   580                 }

   581             } while (true);

   583             NeedChar(')');

   584         }

   586         private void PARSE_Enum(ParseWriter w)

   587         {

   588             string name = "";

   589             if (MatchToken(LexToken.TOKEN_SYMBOL))

   590             {

   591                 name = LexString;

   592             }

   594             w.BeginSection("enum");

   595             w.WritePair("name", name);

   596             w.BeginSection("properties");

   598             w.enumList.Add(name);

   600             NeedChar('{');

   601             bool more_entries = true;

   602             bool need_closebrace = true;

   604             if (MatchToken(LexToken.TOKEN_DOCBLOCK))

   605             {

   606               w.WritePair("doc", LexString);

   607             }

   609             do

   610             {

   611                 if (MatchChar('}'))

   612                 {

   613                     need_closebrace = false;

   614                     break;

   615                 }

   616                 NeedToken(LexToken.TOKEN_SYMBOL);

   617                 name = LexString;

   619                 if (MatchChar('='))

   620                 {

   621                     DiscardUntilCharOrComment(",\n}", true);

   622                 }

   623                 more_entries = MatchChar(',');

   624                 w.WritePair("name", name);

   625                 w.enumTypeList.Add(name);

   626                 if (MatchToken(LexToken.TOKEN_DOCBLOCK))

   627                 {

   628                     w.WritePair("doc", LexString);

   629                 }

   630             } while (more_entries);

   631             if (need_closebrace)

   632             {

   633                 NeedChar('}');

   634             }

   635             NeedChar(';');

   636             w.EndSection();

   637             w.EndSection();

   638         }

   640         private void PARSE_Define(ParseWriter w)

   641         {

   642             /* first, get the symbol name */

   643             NeedToken(LexToken.TOKEN_SYMBOL);

   644             string name = LexString;

   645             string value;

   647             /* read until we hit the end of a line OR a comment */

   648             int endpoint = -1;

   649             for (int i = 0; i < Contents.Length; i++)

   650             {

   651                 if (Contents[i] == '\n')

   652                 {

   653                     endpoint = i;

   654                     break;

   655                 }

   656                 else if (CheckString(i, "/**"))

   657                 {

   658                     endpoint = i;

   659                     break;

   660                 }

   661             }

   663             if (endpoint < 1)

   664             {

   665                 value = "";

   666             }

   667             else

   668             {

   669                 value = Contents.ToString().Substring(0, endpoint);

   670                 Contents.Remove(0, endpoint);

   671             }

   673             /* Write */

   674             w.defineList.Add(name);

   676             w.BeginSection("define");

   677             w.WritePair("name", name);

   678             w.WritePair("linetext", value);

   679             w.EndSection();

   680         }

   682         /* Reads up to a character and discards all text before it.

   683          */

   684         private void DiscardUntilChar(char c)

   685         {

   686             int eol = Contents.ToString().IndexOf(c);

   688             if (eol < 1)

   689             {

   690                 return;

   691             }

   693             Contents.Remove(0, eol);

   694         }

   696         /* Same as DiscardUntilChar, except the discarded

   697          * text is returned first.

   698          */

   699         private string ConsumeUntilChar(char c)

   700         {

   701             int eol = Contents.ToString().IndexOf(c);

   703             if (eol < 1)

   704             {

   705                 return null;

   706             }

   708             string rest = Contents.ToString().Substring(0, eol);

   709             Contents.Remove(0, eol);

   710             return rest;

   711         }

   713         /**

   714          * Returns a parameter's default value.

   715          */

   716         private string ConsumeParamDefValue()

   717         {

   718             int eol = FindEndOfExprLine(0);

   719             if (eol < 1)

   720             {

   721                 return null;

   722             }

   724             string rest = Contents.ToString().Substring(0, eol);

   725             Contents.Remove(0, eol);

   726             return rest;

   727         }

   729         /**

   730          * Discards all data up to a certain character, ignoring

   731          * the contents, unless specified to stop at a doc block comment.

   732          */

   733         private void DiscardUntilCharOrComment(char c, bool stop_at_comment)

   734         {

   735             for (int i = 0; i < Contents.Length; i++)

   736             {

   737                 if (Contents[i] == '\n')

   738                 {

   739                     LineNo++;

   740                 }

   741                 else if (Contents[i] == c)

   742                 {

   743                     if (i > 0)

   744                     {

   745                         Contents.Remove(0, i);

   746                     }

   747                     return;

   748                 }

   749                 else if (stop_at_comment && CheckString(i, "/**"))

   750                 {

   751                     if (i > 0)

   752                     {

   753                         Contents.Remove(0, i);

   754                     }

   755                     return;

   756                 }

   757             }

   758         }

   760        /**

   761         * Discards all data up to a certain character, ignoring

   762         * the contents, unless specified to stop at a doc block comment.

   763         */

   764         private void DiscardUntilCharOrComment(string s, bool stop_at_comment)

   765         {

   766           for (int i = 0; i < Contents.Length; i++)

   767           {

   768             if (Contents[i] == '\n')

   769             {

   770               LineNo++;

   771             }

   772             else if (s.Contains(Contents[i].ToString()))

   773             {

   774               if (i > 0)

   775               {

   776                 Contents.Remove(0, i);

   777               }

   778               return;

   779             }

   780             else if (stop_at_comment && CheckString(i, "/**"))

   781             {

   782               if (i > 0)

   783               {

   784                 Contents.Remove(0, i);

   785               }

   786               return;

   787             }

   788           }

   789         }

   791         /**

   792          * Finds the end of an expression

   793          */

   794         private int FindEndOfExprLine(int start)

   795         {

   796             for (int i = start; i < Contents.Length; i++)

   797             {

   798                 if (Contents[i] == '{')

   799                 {

   800                     if ((i = FindEndOfConsumptionLine('}', start+1)) == -1)

   801                     {

   802                         throw new ParseException("Expected token: } (found none)");

   803                     }

   804                 }

   805                 else if (Contents[i] == ')' || Contents[i] == ',')

   806                 {

   807                     return i;

   808                 }

   809             }

   811             return -1;

   812         }

   814         private int FindEndOfConsumptionLine(char c, int start)

   815         {

   816             for (int i = start; i < Contents.Length; i++)

   817             {

   818                 if (Contents[i] == '{')

   819                 {

   820                     if ((i = FindEndOfConsumptionLine('}', i + 1)) == -1)

   821                     {

   822                         throw new ParseException("Expected token: } (found none)");

   823                     }

   824                 }

   825                 else if (Contents[i] == c)

   826                 {

   827                     return i;

   828                 }

   829             }

   831             return -1;

   832         }

   834         private void Initialize()

   835         {

   836             /* Open file, read it, and close it */

   837             StreamReader sr = File.OpenText(FileName);

   838             Contents = new StringBuilder(sr.ReadToEnd());

   839             sr.Close();

   841             /* Strip comments (may throw an exception) */

   842             preprocess();

   844             /* Lastly, initialize the tokenizer */

   845             Tokenizer.Initialize();

   846         }

   848         /* Consumes a token iff it matches */

   849         private bool MatchToken(LexToken token)

   850         {

   851             LexToken tok = lex();

   852             if (tok != token)

   853             {

   854                 lexpush();

   855                 return false;

   856             }

   857             return true;

   858         }

   860         /* Consumes a token and aborts the parser if there is no match */

   861         private void NeedToken(LexToken token)

   862         {

   863             if (!MatchToken(token))

   864             {

   865                 throw new ParseException("Expected token: " + token);

   866             }

   867         }

   869         /* Consumes a character iff it matches */

   870         private bool MatchChar(char c)

   871         {

   872             if (lex() != LexToken.TOKEN_CHARACTER)

   873             {

   874                 lexpush();

   875                 return false;

   876             }

   877             if (_LexChar != c)

   878             {

   879                 lexpush();

   880                 return false;

   881             }

   882             return true;

   883         }

   885         /* Consumes a character or aborts the parser if there is no match */

   886         private void NeedChar(char c)

   887         {

   888             if (!MatchChar(c))

   889             {

   890                 throw new ParseException("Expected character: " + c);

   891             }

   892         }

   894         /* Saves the lexer state so we don't reparse unnecessarily */

   895         private void lexpush()

   896         {

   897             if (LexPushed)

   898             {

   899                 throw new ParseException("Lexer cannot be pushed twice");

   900             }

   901             LexPushed = true;

   902         }

   904         /* Attempts to match a string against the next stream tokens.

   905          * On success, the token stream is advanced.

   906          */

   907         private bool MatchString(int startPos, string str)

   908         {

   909             if (str.Length > Contents.Length - startPos)

   910             {

   911                 return false;

   912             }

   913             for (int i = 0; i < str.Length; i++)

   914             {

   915                 if (str[i] != Contents[startPos + i])

   916                 {

   917                     return false;

   918                 }

   919             }

   920             Contents.Remove(startPos, str.Length);

   921             return true;

   922         }

   924         /* Attempts to match a string against the next stream tokens */

   925         private bool CheckString(int startPos, string str)

   926         {

   927             if (str.Length > Contents.Length - startPos)

   928             {

   929                 return false;

   930             }

   931             for (int i = 0; i < str.Length; i++)

   932             {

   933                 if (str[i] != Contents[startPos + i])

   934                 {

   935                     return false;

   936                 }

   937             }

   938             return true;

   939         }

   941         /* Ignores a block starting and ending with { and } respectively */

   942         private void ignore_block()

   943         {

   944             /* Hrm, this is tricky.  First, expect an open brace. */

   945             NeedChar('{');

   946             /* Consume the entire function by keeping track of braces */

   947             int numBraces = 1;

   948             bool in_string = false;

   949             bool found_end = false;

   950             for (int i = 0; i < Contents.Length; i++)

   951             {

   952                 if (Contents[i] == '\n')

   953                 {

   954                     LineNo++;

   955                 }

   956                 if (!in_string)

   957                 {

   958                     if (Contents[i] == '"')

   959                     {

   960                         in_string = true;

   961                     }

   962                     else if (Contents[i] == '{')

   963                     {

   964                         numBraces++;

   965                     }

   966                     else if (Contents[i] == '}')

   967                     {

   968                         if (numBraces == 0)

   969                         {

   970                             throw new ParseException("Unable to find a matching open brace");

   971                         }

   972                         if (--numBraces == 0)

   973                         {

   974                             Contents.Remove(0, i + 1);

   975                             found_end = true;

   976                             break;

   977                         }

   978                     }

   979                 }

   980                 else

   981                 {

   982                     /* It is possible to be in a string and at the first character */

   983                     if (Contents[i] == '"' && Contents[i - 1] != '\\')

   984                     {

   985                         in_string = false;

   986                     }

   987                 }

   988             }

   989             if (!found_end)

   990             {

   991                 throw new ParseException("Unable to find a matching close brace");

   992             }

   993         }

   995         /* Strips all non-doc C and C++ style comments. as well as unused pragmas */

   996         private void preprocess()

   997         {

   998             if (Contents.Length == 1)

   999             {

  1000                 return;

  1001             }

  1003             bool in_comment = false;

  1004             bool is_ml = false;

  1005             bool is_doc = false;

  1006             bool is_newline = true;

  1008             for (int i=0; i<Contents.Length; i++)

  1009             {

  1010                 if (Contents[i] == '\n')

  1011                 {

  1012                     is_newline = true;

  1013                 }

  1014                 if (!in_comment)

  1015                 {

  1016                     if (Contents[i] == '/')

  1017                     {

  1018                         if (Contents[i+1] == '/')

  1019                         {

  1020                             in_comment = true;

  1021                             is_ml = false;

  1022                             is_doc = false;

  1023                             is_newline = false;

  1024                         } else if (Contents[i+1] == '*') {

  1025                             /* Detect whether this is a doc comment or not */

  1026                             if (Contents.Length - i >= 3

  1027                                 && Contents[i+2] == '*')

  1028                             {

  1029                                 is_doc = true;

  1030                             } else {

  1031                                 is_doc = false;

  1032                             }

  1033                             in_comment = true;

  1034                             is_ml = true;

  1035                         }

  1036                         if (in_comment && !is_doc)

  1037                         {

  1038                             Contents[i] = ' ';

  1039                             Contents[i+1] = ' ';        /* Just for safety */

  1040                         }

  1041                     }

  1042                     else if (is_newline)

  1043                     {

  1044                         if (Contents[i] == '#')

  1045                         {

  1046                             if (!CheckString(i, "#define")

  1047                                 && !CheckString(i, "#include"))

  1048                             {

  1049                                 /* Ignore! */

  1050                                 Contents[i] = ' ';

  1051                                 in_comment = true;

  1052                                 is_doc = false;

  1053                                 is_ml = false;

  1054                             }

  1055                             is_newline = false;

  1056                         }

  1057                         else if (!Char.IsWhiteSpace(Contents[i]))

  1058                         {

  1059                             is_newline = false;

  1060                         }

  1061                     }

  1062                 } else {

  1063                     if (is_ml)

  1064                     {

  1065                         if (Contents[i] == '*' && Contents[i+1] == '/')

  1066                         {

  1067                             if (!is_doc)

  1068                             {

  1069                                 Contents[i] = ' ';

  1070                                 Contents[i+1] = ' ';

  1071                             }

  1072                             in_comment = false;

  1073                             i++;    /* Don't look at the next character */

  1074                         }

  1075                         else if (!is_doc && Contents[i] != '\n')

  1076                         {

  1077                             /* Erase comment */

  1078                             Contents[i] = ' ';

  1079                         }

  1080                     } else {

  1081                         if (is_newline)

  1082                         {

  1083                             in_comment = false;

  1084                         }

  1085                         else

  1086                         {

  1087                             /* Erase comment */

  1088                             Contents[i] = ' ';

  1089                         }

  1090                     }

  1091                 }

  1092             }

  1094             if (in_comment)

  1095             {

  1096                 throw new ParseException("Multi-line comment has no terminator");

  1097             }

  1098         }

  1100         /* Strips whitespace */

  1101         private void trim_whitespace_pre()

  1102         {

  1103             int spaces = 0;

  1104             while (spaces < Contents.Length

  1105                    && Contents[spaces] != '\0'

  1106                    && Char.IsWhiteSpace(Contents[spaces]))

  1107             {

  1108                 if (Contents[spaces] == '\n')

  1109                 {

  1110                     LineNo++;

  1111                 }

  1112                 spaces++;

  1113             }

  1114             if (spaces > 0)

  1115             {

  1116                 Contents.Remove(0, spaces);

  1117             }

  1118         }

  1120         private void preprocess_line()

  1121         {

  1122             trim_whitespace_pre();

  1123         }

  1125         private bool IsSymbolChar(char c, bool first)

  1126         {

  1127             bool valid = (first ? Char.IsLetter(c) : Char.IsLetterOrDigit(c));

  1128             return (valid || (c == '_'));

  1129         }

  1131         private bool IsEscapedChar(char c)

  1132         {

  1133             return (c == 'n'

  1134                     || c == 'r'

  1135                     || c == 't'

  1136                     || c == 'v'

  1137                     || c == '\\'

  1138                     || c == '"');

  1139         }

  1141         private bool IsNotKeywordChar(char c)

  1142         {

  1143             return (Char.IsWhiteSpace(c)

  1144                     || (c == '-' || c == '+')

  1145                     || (c == '/' || c == '*')

  1146                     || (c == '(' || c == ')')

  1147                     || (c == '=' || c == '%')

  1148                     || (c == '>' || c == '<')

  1149                     || (c == '!' || c == '~')

  1150                     );

  1151         }

  1153         /* Consumes one lexical token and gathers information about it */

  1154         private LexToken lex()

  1155         {

  1156             if (LexPushed)

  1157             {

  1158                 LexPushed = false;

  1159                 return _LastToken;

  1160             }

  1162             /* Number of chars we will be deleting from the input */

  1163             int stripchars = 0;

  1165             /* Clear our state */

  1166             _LastToken = LexToken.TOKEN_NONE;

  1168             /* Remove stuff we don't want from the line */

  1169             preprocess_line();

  1171             if (Contents.Length < 1)

  1172             {

  1173                 _LastToken = LexToken.TOKEN_EOF;

  1174                 return _LastToken;

  1175             }

  1177             /* Get the token list */

  1178             KeywordToken[] tokens = Tokenizer.Tokens;

  1179             for (int i = 0; i < tokens.Length; i++)

  1180             {

  1181                 if (!CheckString(0, tokens[i].keyword))

  1182                 {

  1183                     continue;

  1184                 }

  1185                 int len = tokens[i].keyword.Length;

  1186                 /* Now check to see what the next token is */

  1187                 if (Contents.Length == len

  1188                     || IsNotKeywordChar(Contents[len]))

  1189                 {

  1190                     /* We have a token match! */

  1191                     _LastToken = tokens[i].token;

  1192                     _LexString = null;

  1193                     _LexChar = '\0';

  1194                     stripchars = len;

  1195                     break;

  1196                 }

  1197             }

  1199             if (_LastToken == LexToken.TOKEN_NONE)

  1200             {

  1201                 /* See if we can try to read a symbol */

  1202                 if (IsSymbolChar(Contents[0], true))

  1203                 {

  1204                     int characters = 1;

  1205                     while (Contents[characters] != '\0'

  1206                            && IsSymbolChar(Contents[characters], false))

  1207                     {

  1208                         characters++;

  1209                     }

  1210                     stripchars = characters;

  1211                     /* We're done! See what's next.. */

  1212                     if (Contents[characters] == ':')

  1213                     {

  1214                         _LastToken = LexToken.TOKEN_LABEL;

  1215                         stripchars++;

  1216                     }

  1217                     else

  1218                     {

  1219                         _LastToken = LexToken.TOKEN_SYMBOL;

  1220                     }

  1221                     _LexString = Contents.ToString().Substring(0, characters);

  1222                     _LexChar = _LexString[0];

  1223                 }/*

  1224                 else if (Contents[0] == '"')

  1225                 {

  1226                     bool reached_end = false;

  1227                     StringBuilder builder = new StringBuilder();

  1228                     for (int i = 1; i < Contents.Length; i++)

  1229                     {

  1230                         if (i < Contents.Length - 1

  1231                             && Contents[i] == '\\'

  1232                             && IsEscapedChar(Contents[i+1]))

  1233                         {

  1234                             i++;

  1235                         }

  1236                         else if (Contents[i] == '"')

  1237                         {

  1238                             reached_end = true;

  1239                             break;

  1240                         }

  1241                         builder.Append(Contents[i]);

  1242                     }

  1243                     if (!reached_end)

  1244                     {

  1245                         throw new ParseException("Expected end of string; none found");

  1246                     }

  1247                     _LexString = builder.ToString();

  1248                     _LexChar = _LexString.Length > 0 ? _LexString[0] : '\0';

  1249                 }*/

  1250                 else if (Contents[0] == '\'')

  1251                 {

  1252                     char c = '\0';

  1253                     if (Contents.Length < 3)

  1254                     {

  1255                         throw new ParseException("Expected end of character; none found");

  1256                     }

  1257                     else if (Contents[1] == '\\')

  1258                     {

  1259                         if (Contents.Length < 4 || !IsEscapedChar(Contents[2]))

  1260                         {

  1261                             throw new ParseException("Expected end of character; none found");

  1262                         }

  1263                         if (Contents[2] == 'r')

  1264                         {

  1265                             c = '\r';

  1266                         }

  1267                         else if (Contents[2] == 'n')

  1268                         {

  1269                             c = '\n';

  1270                         }

  1271                         else if (Contents[2] == 't')

  1272                         {

  1273                             c = '\t';

  1274                         }

  1275                         else if (Contents[2] == 'v')

  1276                         {

  1277                             c = '\v';

  1278                         }

  1279                         else if (Contents[2] == '\\')

  1280                         {

  1281                             c = '\\';

  1282                         }

  1283                         else if (Contents[2] == '"')

  1284                         {

  1285                             c = '"';

  1286                         }

  1287                         stripchars = 4;

  1288                     }

  1289                     else

  1290                     {

  1291                         c = Contents[1];

  1292                         stripchars = 3;

  1293                     }

  1294                     _LexString = c.ToString();

  1295                     _LexChar = c;

  1296                     _LastToken = LexToken.TOKEN_QUOTCHAR;

  1297                 }

  1298                 else if (CheckString(0, "/**"))

  1299                 {

  1300                     int endpoint = 0;

  1301                     for (int i=3; i<Contents.Length-1; i++)

  1302                     {

  1303                         if (Contents[i] == '\n')

  1304                         {

  1305                             LineNo++;

  1306                         }

  1307                         else if (Contents[i] == '*' && Contents[i+1] == '/')

  1308                         {

  1309                             endpoint = i+1;

  1310                             break;

  1311                         }

  1312                     }

  1313                     if (endpoint == 0)

  1314                     {

  1315                         throw new ParseException("Expected end of multi-line comment; found end of file");

  1316                     }

  1317                     /* We have to factor one more character in because we're zero based */

  1318                     endpoint++;

  1319                     _LastToken = LexToken.TOKEN_DOCBLOCK;

  1320                     _LexString = Contents.ToString().Substring(0, endpoint);

  1321                     _LexChar = _LexString[0];

  1322                     stripchars = endpoint;

  1323                 }

  1324                 else

  1325                 {

  1326                     _LastToken = LexToken.TOKEN_CHARACTER;

  1327                     _LexString = Contents[0].ToString();

  1328                     _LexChar = Contents[0];

  1329                     stripchars = 1;

  1330                 }

  1331             }

  1333             /* Strip N chars */

  1334             if (stripchars > 0)

  1335             {

  1336                 Contents.Remove(0, stripchars);

  1337             }

  1339             return _LastToken;

  1340         }

  1341     }

  1342 }

author	Peace-Maker <januk@freenet.de>
	Tue Mar 19 11:25:04 2013 -0400 (2013-03-19)
changeset 3859	4e43bcc83a54
parent 2765	ace14b1768dc
permissions	-rw-r--r--