dvander@824: using System;
dvander@824: using System.Collections.Generic;
dvander@824: using System.IO;
dvander@824: using System.Text;
dvander@824: 
dvander@824: namespace incparser
dvander@824: {
dvander@824:     enum LexToken
dvander@824:     {
dvander@824:         TOKEN_NONE,
dvander@824:         TOKEN_INCLUDE,
dvander@824:         TOKEN_DEFINE,
dvander@824:         TOKEN_CONST,
dvander@824:         TOKEN_STRUCT,
dvander@824:         TOKEN_FUNCTAG,
dvander@824:         TOKEN_FUNCENUM,
dvander@824:         TOKEN_ENUM,
dvander@824:         TOKEN_NATIVE,
dvander@824:         TOKEN_STOCK,
dvander@824:         TOKEN_PUBLIC,
dvander@824:         TOKEN_FORWARD,
dvander@824:         TOKEN_QUOTCHAR,
dvander@824:         TOKEN_OPERATOR,
dvander@824:         TOKEN_CHARACTER,
dvander@824:         /*TOKEN_STRING,*/
dvander@824:         TOKEN_LABEL,
dvander@824:         TOKEN_SYMBOL,
dvander@824:         TOKEN_DOCBLOCK,
dvander@824:         TOKEN_EOF,
dvander@824:     }
dvander@824: 
dvander@824:     class ParseException : System.Exception
dvander@824:     {
dvander@824:         public ParseException(string message) : base(message)
dvander@824:         {
dvander@824:         }
dvander@824:     };
dvander@824:          
dvander@824:     class KeywordToken
dvander@824:     {
dvander@824:         public KeywordToken(string k, LexToken t)
dvander@824:         {
dvander@824:             keyword = k;
dvander@824:             token = t;
dvander@824:         }
dvander@824:         public string keyword;
dvander@824:         public LexToken token;
dvander@824:     };
dvander@824: 
dvander@824:     class Tokenizer
dvander@824:     {
dvander@824:         private static bool s_initialized = false;
dvander@824:         private static int s_position = 0;
dvander@824:         private static KeywordToken [] s_tokens = null;
dvander@824:         public static KeywordToken[] Tokens = null;
dvander@824: 
dvander@824:         public static void Initialize()
dvander@824:         {
dvander@824:             if (s_initialized)
dvander@824:             {
dvander@824:                 return;
dvander@824:             }
dvander@824:             s_tokens = new KeywordToken[12];
dvander@824:             s_initialized = true;
dvander@824:             s_position = 0;
dvander@824: 
dvander@824:             AddToken("#include", LexToken.TOKEN_INCLUDE);
dvander@824:             AddToken("#define", LexToken.TOKEN_DEFINE);
dvander@824:             AddToken("native", LexToken.TOKEN_NATIVE);
dvander@824:             AddToken("functag", LexToken.TOKEN_FUNCTAG);
dvander@824:             AddToken("funcenum", LexToken.TOKEN_FUNCENUM);
dvander@824:             AddToken("enum", LexToken.TOKEN_ENUM);
dvander@824:             AddToken("stock", LexToken.TOKEN_STOCK);
dvander@824:             AddToken("public", LexToken.TOKEN_PUBLIC);
dvander@824:             AddToken("forward", LexToken.TOKEN_FORWARD);
dvander@824:             AddToken("const", LexToken.TOKEN_CONST);
dvander@824:             AddToken("struct", LexToken.TOKEN_STRUCT);
dvander@824:             AddToken("operator", LexToken.TOKEN_OPERATOR);
dvander@824: 
dvander@824:             Tokens = s_tokens;
dvander@824:         }
dvander@824: 
dvander@824:         private static void AddToken(string lex, LexToken tok)
dvander@824:         {
dvander@824:             s_tokens[s_position] = new KeywordToken(lex, tok);
dvander@824:             s_position++;
dvander@824:         }
dvander@824:     };
dvander@824: 
dvander@824:     class IncParser
dvander@824:     {
dvander@824:         private string FileName;
dvander@824:         private StringBuilder Contents;
dvander@824:         private bool LexPushed;
dvander@824:         private LexToken _LastToken;
dvander@824:         private string _LexString;
dvander@824:         private char _LexChar;
dvander@824:         private uint LineNo;
dvander@824: 
dvander@824:         public string LexString
dvander@824:         {
dvander@824:             get
dvander@824:             {
dvander@824:                 return _LexString;
dvander@824:             }
dvander@824:         }
dvander@824: 
dvander@824:         public char LexChar
dvander@824:         {
dvander@824:             get
dvander@824:             {
dvander@824:                 return _LexChar;
dvander@824:             }
dvander@824:         }
dvander@824: 
dvander@824:         public uint GetLineNumber()
dvander@824:         {
dvander@824:             return LineNo;
dvander@824:         }
dvander@824: 
dvander@824:         public IncParser(string file)
dvander@824:         {
dvander@824:             FileName = file;
dvander@824: 
dvander@824:             /* Clear out lexer stuff */
dvander@824:             LexPushed = false;
dvander@824:             _LastToken = LexToken.TOKEN_NONE;
dvander@824:             _LexString = null;
dvander@824:             _LexChar = '\0';
dvander@824:             LineNo = 1;
dvander@824: 
dvander@824:             /* Initialize; this can throw an exception */
dvander@824:             Initialize();
dvander@824:         }
dvander@824: 
pred@2019:          public void Parse(ParseWriter w)
dvander@824:         {
dvander@824:             w.BeginSection(System.IO.Path.GetFileName(FileName));
dvander@824:             LexToken tok = LexToken.TOKEN_NONE;
dvander@824:             while ((tok = lex()) != LexToken.TOKEN_EOF)
dvander@824:             {
dvander@824:                 switch (tok)
dvander@824:                 {
dvander@824:                     case LexToken.TOKEN_DOCBLOCK:
dvander@824:                         {
dvander@824:                             w.WritePair("doc", LexString);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_DEFINE:
dvander@824:                         {
dvander@824:                             PARSE_Define(w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_ENUM:
dvander@824:                         {
dvander@824:                             PARSE_Enum(w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_FORWARD:
dvander@824:                         {
dvander@824:                             PARSE_Function(tok, w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_NATIVE:
dvander@824:                         {
dvander@824:                             PARSE_Function(tok, w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_STOCK:
dvander@824:                         {
dvander@824:                             PARSE_Function(tok, w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_FUNCTAG:
dvander@824:                         {
dvander@824:                             PARSE_FuncTag(w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_FUNCENUM:
dvander@824:                         {
dvander@824:                             PARSE_FuncEnum(w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_INCLUDE:
dvander@824:                         {
dvander@824:                             PARSE_Include(w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_STRUCT:
dvander@824:                         {
dvander@824:                             PARSE_Struct(w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     case LexToken.TOKEN_PUBLIC:
dvander@824:                         {
dvander@824:                             PARSE_Public(w);
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     default:
dvander@824:                         {
pred@2019:                             if (_LexString == "static" || _LexString == "new")
pred@2019:                             {
pred@2019:                                 PARSE_Public(w);
pred@2019:                                 break;
pred@2019:                             }
pred@2019:                             throw new ParseException("Unrecognized token: " + tok + " " + _LexString);
dvander@824:                         }
dvander@824:                 }
dvander@824:             }
dvander@824:             w.EndSection();
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_Public(ParseWriter w)
dvander@824:         {
pred@2019:             MatchToken(LexToken.TOKEN_CONST);
pred@2019: 
dvander@824:             /* Eat up an optional tag */
dvander@824:             MatchToken(LexToken.TOKEN_LABEL);
pred@2019: 
dvander@824:             /* Eat up a name */
dvander@824:             NeedToken(LexToken.TOKEN_SYMBOL);
dvander@824: 
pred@2019:             while (MatchChar('['))
pred@2019:             {
pred@2019:                 DiscardUntilChar(']');
pred@2019:                 NeedChar(']');
pred@2019:             }
pred@2019: 
dvander@824:             if (MatchChar('='))
dvander@824:             {
dvander@824:                 ignore_block();
dvander@824:             }
pred@2019:             else if (MatchChar('(') && MatchChar(')'))
pred@2019:             {
pred@2019:                 ignore_block();
pred@2019:                 return;
pred@2019:             }
dvander@824: 
dvander@824:             NeedChar(';');
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_Struct(ParseWriter w)
dvander@824:         {
dvander@824:             /* For now, we completely ignore these (they're not written to the output) */
dvander@824:             NeedToken(LexToken.TOKEN_SYMBOL);
pred@2019:             w.structList.Add(LexString);
dvander@824:             NeedChar('{');
dvander@824: 
dvander@824:             bool need_closebrace = true;
dvander@824:             bool more_entries = true;
dvander@824:             do
dvander@824:             {
dvander@824:                 if (MatchChar('}'))
dvander@824:                 {
dvander@824:                     need_closebrace = false;
dvander@824:                     break;
dvander@824:                 }
dvander@824:                 MatchToken(LexToken.TOKEN_CONST);
dvander@824:                 MatchToken(LexToken.TOKEN_LABEL);
dvander@824:                 NeedToken(LexToken.TOKEN_SYMBOL);
dvander@824:                 while (MatchChar('['))
dvander@824:                 {
dvander@824:                     DiscardUntilChar(']');
dvander@824:                     NeedChar(']');
dvander@824:                 }
dvander@824:                 more_entries = MatchChar(',');
dvander@824:                 MatchToken(LexToken.TOKEN_DOCBLOCK);
dvander@824:                 if (!more_entries)
dvander@824:                 {
dvander@824:                     break;
dvander@824:                 }
dvander@824:             } while (true);
dvander@824:             if (need_closebrace)
dvander@824:             {
dvander@824:                 NeedChar('}');
dvander@824:             }
dvander@824:             NeedChar(';');
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_Include(ParseWriter w)
dvander@824:         {
dvander@824:             /* read until the end of the line */
dvander@824:             int index = Contents.ToString().IndexOf('\n');
dvander@824:             string value;
dvander@824:             if (index == -1)
dvander@824:             {
dvander@824:                 value = Contents.ToString();
dvander@824:                 Contents.Remove(0, value.Length);
dvander@824:             }
dvander@824:             else if (index == 0)
dvander@824:             {
dvander@824:                 value = "";
dvander@824:             }
dvander@824:             else
dvander@824:             {
dvander@824:                 value = Contents.ToString().Substring(0, index);
dvander@824:                 Contents.Remove(0, index);
dvander@824:             }
dvander@824: 
dvander@824:             /* Strip whitespace */
dvander@824:             value = value.Trim('\r', '\t', ' ');
dvander@824: 
dvander@824:             /* Write */
dvander@824:             w.BeginSection("include");
dvander@824:             w.WritePair("name", value);
dvander@824:             w.EndSection();
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_FuncEnum(ParseWriter w)
dvander@824:         {
dvander@824:             w.BeginSection("funcenum");
dvander@824: 
dvander@824:             /* Get the functag name */
dvander@824:             NeedToken(LexToken.TOKEN_SYMBOL);
dvander@824:             w.WritePair("name", LexString);
pred@2019:             w.funcenumList.Add(LexString);
dvander@824: 
dvander@824:             NeedChar('{');
dvander@824: 
dvander@824:             bool need_closebrace = true;
dvander@824:             do
dvander@824:             {
dvander@824:                 /* Shortcut out? */
dvander@824:                 if (MatchChar('}'))
dvander@824:                 {
dvander@824:                     need_closebrace = false;
dvander@824:                     break;
dvander@824:                 }
dvander@824: 
dvander@824:                 /* Start function section */
dvander@824:                 w.BeginSection("function");
dvander@824: 
dvander@824:                 if (MatchToken(LexToken.TOKEN_DOCBLOCK))
dvander@824:                 {
dvander@824:                     w.WritePair("doc", LexString);
dvander@824:                 }
dvander@824:                 
dvander@824:                 /* Get the return tag */
dvander@824:                 if (MatchToken(LexToken.TOKEN_LABEL))
dvander@824:                 {
dvander@824:                     w.WritePair("return", LexString);
dvander@824:                 }
dvander@824: 
dvander@824:                 /* Get function type */
dvander@824:                 if (MatchToken(LexToken.TOKEN_PUBLIC))
dvander@824:                 {
dvander@824:                     w.WritePair("type", "public");
dvander@824:                 }
dvander@824:                 else if (MatchToken(LexToken.TOKEN_STOCK))
dvander@824:                 {
dvander@824:                     w.WritePair("type", "stock");
dvander@824:                 }
dvander@824:                 
dvander@824:                 /* Parse the parameters */
dvander@824:                 ParseParameters(w);
dvander@824: 
dvander@824:                 /* End the section */
dvander@824:                 w.EndSection();
dvander@824: 
dvander@824:                 if (!MatchChar(','))
dvander@824:                 {
dvander@824:                     break;
dvander@824:                 }
dvander@824:             } while (true);
dvander@824: 
dvander@824:             if (need_closebrace)
dvander@824:             {
dvander@824:                 NeedChar('}');
dvander@824:             }
dvander@824:             NeedChar(';');
dvander@824: 
dvander@824:             w.EndSection();
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_FuncTag(ParseWriter w)
dvander@824:         {
dvander@824:             w.BeginSection("functag");
dvander@824: 
dvander@824:             if (MatchToken(LexToken.TOKEN_PUBLIC))
dvander@824:             {
dvander@824:                 w.WritePair("type", "public");
dvander@824:             }
dvander@824:             else if (MatchToken(LexToken.TOKEN_STOCK))
dvander@824:             {
dvander@824:                 w.WritePair("type", "stock");
dvander@824:             }
dvander@824: 
pred@2765:             if (MatchToken(LexToken.TOKEN_LABEL))
pred@2765:             {
pred@2765:                 w.WritePair("return", LexString);
pred@2765:             }
pred@2765: 
pred@2765:             /* Get the functag name */
pred@2765:             NeedToken(LexToken.TOKEN_SYMBOL);
pred@2765:             w.WritePair("name", LexString);
pred@2765:             w.functagList.Add(LexString);
pred@2765: 
dvander@824:             ParseParameters(w);
dvander@824:             NeedChar(';');
dvander@824:             
dvander@824:             w.EndSection();
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_Function(LexToken tok, ParseWriter w)
dvander@824:         {
dvander@824:             string tag="", name;
dvander@824: 
dvander@824:             /* Get the return value */
dvander@824:             if (MatchToken(LexToken.TOKEN_LABEL))
dvander@824:             {
dvander@824:                 tag = LexString;
dvander@824:             }
dvander@824: 
dvander@824:             if (MatchToken(LexToken.TOKEN_OPERATOR))
dvander@824:             {
dvander@824:                 /* Check if we're some sort of invalid function */
dvander@824:                 DiscardUntilCharOrComment('(', false);
dvander@824:                 ParseParameters(null);
dvander@824:                 if (tok == LexToken.TOKEN_STOCK)
dvander@824:                 {
dvander@824:                     ignore_block();
dvander@824:                 }
dvander@824:                 else if (tok == LexToken.TOKEN_NATIVE || tok == LexToken.TOKEN_FORWARD)
dvander@824:                 {
dvander@824:                     if (MatchChar('='))
dvander@824:                     {
dvander@824:                         DiscardUntilCharOrComment(';', false);
dvander@824:                     }
dvander@824:                     NeedChar(';');
dvander@824:                 }
dvander@824:                 return;
dvander@824:             }
dvander@824: 
dvander@824:             /* Get the name */
dvander@824:             NeedToken(LexToken.TOKEN_SYMBOL);
dvander@824:             name = LexString;
dvander@824: 
dvander@824:             PARSE_Function(tok, w, tag, name);
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_Function(LexToken tok, ParseWriter w, string tag, string name)
dvander@824:         {
dvander@824:             if (tok == LexToken.TOKEN_FORWARD)
dvander@824:             {
dvander@824:                 w.BeginSection("forward");
pred@2019:                 w.forwardList.Add(name);
dvander@824:             }
dvander@824:             else if (tok == LexToken.TOKEN_NATIVE)
dvander@824:             {
dvander@824:                 w.BeginSection("native");
pred@2019:                 w.nativeList.Add(name);
dvander@824:             }
dvander@824:             else if (tok == LexToken.TOKEN_STOCK)
dvander@824:             {
dvander@824:                 w.BeginSection("stock");
pred@2019:                 w.stockList.Add(name);
dvander@824:             }
dvander@824: 
dvander@824:             w.WritePair("name", name);
dvander@824: 
dvander@824:             if (tag.Length > 0)
dvander@824:             {
dvander@824:                 w.WritePair("return", tag);
dvander@824:             }
dvander@824: 
dvander@824:             ParseParameters(w);
dvander@824: 
dvander@824:             if (tok == LexToken.TOKEN_STOCK)
dvander@824:             {
dvander@824:                 ignore_block();
dvander@824:             }
dvander@824:             else
dvander@824:             {
dvander@824:                 /* Make sure there's a semicolon */
dvander@824:                 if (MatchChar('='))
dvander@824:                 {
dvander@824:                     DiscardUntilCharOrComment(';', false);
dvander@824:                 }
dvander@824:                 NeedChar(';');
dvander@824:             }
dvander@824: 
dvander@824:             w.EndSection();
dvander@824:         }
dvander@824: 
dvander@824:         private void ParseParameters(ParseWriter w)
dvander@824:         {
dvander@824:             NeedChar('(');
dvander@824:             string extra;
dvander@824:             do
dvander@824:             {
dvander@824:                 if (MatchChar(')'))
dvander@824:                 {
dvander@824:                     return;
dvander@824:                 }
dvander@824: 
dvander@824:                 /* Start the parameter */
dvander@824:                 if (w != null)
dvander@824:                 {
dvander@824:                     w.BeginSection("parameter");
dvander@824:                 }
dvander@824: 
dvander@824:                 /* Check for a 'const' token */
dvander@824:                 if (MatchToken(LexToken.TOKEN_CONST) && w != null)
dvander@824:                 {
dvander@824:                     w.WritePair("const", "true");
dvander@824:                 }
dvander@824: 
dvander@824:                 /* Check if it's by reference */
dvander@824:                 if (MatchChar('&') && w != null)
dvander@824:                 {
dvander@824:                     w.WritePair("byref", "true");
dvander@824:                 }
dvander@824: 
dvander@824:                 /* Check for a tag */
dvander@824:                 if (MatchToken(LexToken.TOKEN_LABEL) && w != null)
dvander@824:                 {
dvander@824:                     w.WritePair("tag", LexString);
dvander@824:                 }
dvander@824: 
dvander@824:                 /* Get the symbol and write it */
dvander@824:                 if (MatchString(0, "...") && w != null)
dvander@824:                 {
dvander@824:                     w.WritePair("name", "...");
dvander@824:                 }
dvander@824:                 else
dvander@824:                 {
dvander@824:                     NeedToken(LexToken.TOKEN_SYMBOL);
dvander@824:                     if (w != null)
dvander@824:                     {
dvander@824:                         w.WritePair("name", LexString);
dvander@824:                     }
dvander@824:                 }
dvander@824: 
dvander@824:                 /* Check if we have a default value yet */
dvander@824:                 bool default_value = false;
dvander@824:                 if (MatchChar('='))
dvander@824:                 {
dvander@824:                     default_value = true;
dvander@824:                 }
dvander@824:                 else if (MatchChar('['))
dvander@824:                 {
dvander@824:                     extra = "";
dvander@824:                     string temp;
dvander@824:                     do
dvander@824:                     {
dvander@824:                         extra += "[";
dvander@824:                         temp = ConsumeUntilChar(']');
dvander@824:                         if (temp != null)
dvander@824:                         {
dvander@824:                             extra += temp;
dvander@824:                         }
dvander@824:                         extra += "]";
dvander@824:                         NeedChar(']');
dvander@824:                     } while (MatchChar('['));
dvander@824:                     w.WritePair("decoration", extra);
dvander@824:                 }
dvander@824: 
dvander@824:                 /* If we have a default value, get it */
dvander@824:                 if (default_value || MatchChar('='))
dvander@824:                 {
dvander@824:                     if ((extra = ConsumeParamDefValue()) == null)
dvander@824:                     {
dvander@824:                         throw new ParseException("Expected a default value; found none");
dvander@824:                     }
dvander@824:                     if (w != null)
dvander@824:                     {
dvander@824:                         w.WritePair("defval", extra);
dvander@824:                     }
dvander@824:                 }
dvander@824: 
dvander@824:                 /* End the parameter */
dvander@824:                 if (w != null)
dvander@824:                 {
dvander@824:                     w.EndSection();
dvander@824:                 }
dvander@824: 
dvander@824:                 if (!MatchChar(','))
dvander@824:                 {
dvander@824:                     break;
dvander@824:                 }
dvander@824:             } while (true);
dvander@824: 
dvander@824:             NeedChar(')');
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_Enum(ParseWriter w)
dvander@824:         {
dvander@824:             string name = "";
dvander@824:             if (MatchToken(LexToken.TOKEN_SYMBOL))
dvander@824:             {
dvander@824:                 name = LexString;
dvander@824:             }
dvander@824: 
dvander@824:             w.BeginSection("enum");
dvander@824:             w.WritePair("name", name);
dvander@824:             w.BeginSection("properties");
dvander@824: 
pred@2019:             w.enumList.Add(name);
pred@2019: 
dvander@824:             NeedChar('{');
dvander@824:             bool more_entries = true;
dvander@824:             bool need_closebrace = true;
pred@2019: 
pred@2019:             if (MatchToken(LexToken.TOKEN_DOCBLOCK))
pred@2019:             {
pred@2019:               w.WritePair("doc", LexString);
pred@2019:             }
pred@2019: 
dvander@824:             do
dvander@824:             {
dvander@824:                 if (MatchChar('}'))
dvander@824:                 {
dvander@824:                     need_closebrace = false;
dvander@824:                     break;
dvander@824:                 }
dvander@824:                 NeedToken(LexToken.TOKEN_SYMBOL);
dvander@824:                 name = LexString;
pred@2019:  
dvander@824:                 if (MatchChar('='))
dvander@824:                 {
pred@2019:                     DiscardUntilCharOrComment(",\n}", true);
dvander@824:                 }
dvander@824:                 more_entries = MatchChar(',');
dvander@824:                 w.WritePair("name", name);
pred@2019:                 w.enumTypeList.Add(name);
dvander@824:                 if (MatchToken(LexToken.TOKEN_DOCBLOCK))
dvander@824:                 {
dvander@824:                     w.WritePair("doc", LexString);
dvander@824:                 }
dvander@824:             } while (more_entries);
dvander@824:             if (need_closebrace)
dvander@824:             {
dvander@824:                 NeedChar('}');
dvander@824:             }
dvander@824:             NeedChar(';');
dvander@824:             w.EndSection();
dvander@824:             w.EndSection();
dvander@824:         }
dvander@824: 
dvander@824:         private void PARSE_Define(ParseWriter w)
dvander@824:         {
dvander@824:             /* first, get the symbol name */
dvander@824:             NeedToken(LexToken.TOKEN_SYMBOL);
dvander@824:             string name = LexString;
dvander@824:             string value;
dvander@824: 
dvander@824:             /* read until we hit the end of a line OR a comment */
dvander@824:             int endpoint = -1;
dvander@824:             for (int i = 0; i < Contents.Length; i++)
dvander@824:             {
dvander@824:                 if (Contents[i] == '\n')
dvander@824:                 {
dvander@824:                     endpoint = i;
dvander@824:                     break;
dvander@824:                 }
dvander@824:                 else if (CheckString(i, "/**"))
dvander@824:                 {
dvander@824:                     endpoint = i;
dvander@824:                     break;
dvander@824:                 }
dvander@824:             }
dvander@824: 
dvander@824:             if (endpoint < 1)
dvander@824:             {
dvander@824:                 value = "";
dvander@824:             }
dvander@824:             else
dvander@824:             {
dvander@824:                 value = Contents.ToString().Substring(0, endpoint);
dvander@824:                 Contents.Remove(0, endpoint);
dvander@824:             }
dvander@824: 
dvander@824:             /* Write */
pred@2019:             w.defineList.Add(name);
pred@2019: 
dvander@824:             w.BeginSection("define");
dvander@824:             w.WritePair("name", name);
dvander@824:             w.WritePair("linetext", value);
dvander@824:             w.EndSection();
dvander@824:         }
dvander@824: 
dvander@824:         /* Reads up to a character and discards all text before it.
dvander@824:          */
dvander@824:         private void DiscardUntilChar(char c)
dvander@824:         {
dvander@824:             int eol = Contents.ToString().IndexOf(c);
dvander@824: 
dvander@824:             if (eol < 1)
dvander@824:             {
dvander@824:                 return;
dvander@824:             }
dvander@824: 
dvander@824:             Contents.Remove(0, eol);
dvander@824:         }
dvander@824: 
dvander@824:         /* Same as DiscardUntilChar, except the discarded
dvander@824:          * text is returned first.
dvander@824:          */
dvander@824:         private string ConsumeUntilChar(char c)
dvander@824:         {
dvander@824:             int eol = Contents.ToString().IndexOf(c);
dvander@824: 
dvander@824:             if (eol < 1)
dvander@824:             {
dvander@824:                 return null;
dvander@824:             }
dvander@824: 
dvander@824:             string rest = Contents.ToString().Substring(0, eol);
dvander@824:             Contents.Remove(0, eol);
dvander@824:             return rest;
dvander@824:         }
dvander@824: 
dvander@824:         /**
dvander@824:          * Returns a parameter's default value.
dvander@824:          */
dvander@824:         private string ConsumeParamDefValue()
dvander@824:         {
dvander@824:             int eol = FindEndOfExprLine(0);
dvander@824:             if (eol < 1)
dvander@824:             {
dvander@824:                 return null;
dvander@824:             }
dvander@824: 
dvander@824:             string rest = Contents.ToString().Substring(0, eol);
dvander@824:             Contents.Remove(0, eol);
dvander@824:             return rest;
dvander@824:         }
dvander@824: 
dvander@824:         /**
dvander@824:          * Discards all data up to a certain character, ignoring
dvander@824:          * the contents, unless specified to stop at a doc block comment.
dvander@824:          */
dvander@824:         private void DiscardUntilCharOrComment(char c, bool stop_at_comment)
dvander@824:         {
dvander@824:             for (int i = 0; i < Contents.Length; i++)
dvander@824:             {
dvander@824:                 if (Contents[i] == '\n')
dvander@824:                 {
dvander@824:                     LineNo++;
dvander@824:                 }
dvander@824:                 else if (Contents[i] == c)
dvander@824:                 {
dvander@824:                     if (i > 0)
dvander@824:                     {
dvander@824:                         Contents.Remove(0, i);
dvander@824:                     }
dvander@824:                     return;
dvander@824:                 }
dvander@824:                 else if (stop_at_comment && CheckString(i, "/**"))
dvander@824:                 {
dvander@824:                     if (i > 0)
dvander@824:                     {
dvander@824:                         Contents.Remove(0, i);
dvander@824:                     }
dvander@824:                     return;
dvander@824:                 }
dvander@824:             }
dvander@824:         }
dvander@824: 
pred@2019:        /**
pred@2019:         * Discards all data up to a certain character, ignoring
pred@2019:         * the contents, unless specified to stop at a doc block comment.
pred@2019:         */
pred@2019:         private void DiscardUntilCharOrComment(string s, bool stop_at_comment)
pred@2019:         {
pred@2019:           for (int i = 0; i < Contents.Length; i++)
pred@2019:           {
pred@2019:             if (Contents[i] == '\n')
pred@2019:             {
pred@2019:               LineNo++;
pred@2019:             }
pred@2019:             else if (s.Contains(Contents[i].ToString()))
pred@2019:             {
pred@2019:               if (i > 0)
pred@2019:               {
pred@2019:                 Contents.Remove(0, i);
pred@2019:               }
pred@2019:               return;
pred@2019:             }
pred@2019:             else if (stop_at_comment && CheckString(i, "/**"))
pred@2019:             {
pred@2019:               if (i > 0)
pred@2019:               {
pred@2019:                 Contents.Remove(0, i);
pred@2019:               }
pred@2019:               return;
pred@2019:             }
pred@2019:           }
pred@2019:         }
pred@2019: 
dvander@824:         /**
dvander@824:          * Finds the end of an expression
dvander@824:          */
dvander@824:         private int FindEndOfExprLine(int start)
dvander@824:         {
dvander@824:             for (int i = start; i < Contents.Length; i++)
dvander@824:             {
dvander@824:                 if (Contents[i] == '{')
dvander@824:                 {
dvander@824:                     if ((i = FindEndOfConsumptionLine('}', start+1)) == -1)
dvander@824:                     {
dvander@824:                         throw new ParseException("Expected token: } (found none)");
dvander@824:                     }
dvander@824:                 }
dvander@824:                 else if (Contents[i] == ')' || Contents[i] == ',')
dvander@824:                 {
dvander@824:                     return i;
dvander@824:                 }
dvander@824:             }
dvander@824: 
dvander@824:             return -1;
dvander@824:         }
dvander@824: 
dvander@824:         private int FindEndOfConsumptionLine(char c, int start)
dvander@824:         {
dvander@824:             for (int i = start; i < Contents.Length; i++)
dvander@824:             {
dvander@824:                 if (Contents[i] == '{')
dvander@824:                 {
dvander@824:                     if ((i = FindEndOfConsumptionLine('}', i + 1)) == -1)
dvander@824:                     {
dvander@824:                         throw new ParseException("Expected token: } (found none)");
dvander@824:                     }
dvander@824:                 }
dvander@824:                 else if (Contents[i] == c)
dvander@824:                 {
dvander@824:                     return i;
dvander@824:                 }
dvander@824:             }
dvander@824: 
dvander@824:             return -1;
dvander@824:         }
dvander@824: 
dvander@824:         private void Initialize()
dvander@824:         {
dvander@824:             /* Open file, read it, and close it */
dvander@824:             StreamReader sr = File.OpenText(FileName);
dvander@824:             Contents = new StringBuilder(sr.ReadToEnd());
dvander@824:             sr.Close();
dvander@824: 
dvander@824:             /* Strip comments (may throw an exception) */
dvander@824:             preprocess();
dvander@824: 
dvander@824:             /* Lastly, initialize the tokenizer */
dvander@824:             Tokenizer.Initialize();
dvander@824:         }
dvander@824: 
dvander@824:         /* Consumes a token iff it matches */
dvander@824:         private bool MatchToken(LexToken token)
dvander@824:         {
dvander@824:             LexToken tok = lex();
dvander@824:             if (tok != token)
dvander@824:             {
dvander@824:                 lexpush();
dvander@824:                 return false;
dvander@824:             }
dvander@824:             return true;
dvander@824:         }
dvander@824: 
dvander@824:         /* Consumes a token and aborts the parser if there is no match */
dvander@824:         private void NeedToken(LexToken token)
dvander@824:         {
dvander@824:             if (!MatchToken(token))
dvander@824:             {
dvander@824:                 throw new ParseException("Expected token: " + token);
dvander@824:             }
dvander@824:         }
dvander@824: 
dvander@824:         /* Consumes a character iff it matches */
dvander@824:         private bool MatchChar(char c)
dvander@824:         {
dvander@824:             if (lex() != LexToken.TOKEN_CHARACTER)
dvander@824:             {
dvander@824:                 lexpush();
dvander@824:                 return false;
dvander@824:             }
dvander@824:             if (_LexChar != c)
dvander@824:             {
dvander@824:                 lexpush();
dvander@824:                 return false;
dvander@824:             }
dvander@824:             return true;
dvander@824:         }
dvander@824: 
dvander@824:         /* Consumes a character or aborts the parser if there is no match */
dvander@824:         private void NeedChar(char c)
dvander@824:         {
dvander@824:             if (!MatchChar(c))
dvander@824:             {
dvander@824:                 throw new ParseException("Expected character: " + c);
dvander@824:             }
dvander@824:         }
dvander@824:         
dvander@824:         /* Saves the lexer state so we don't reparse unnecessarily */
dvander@824:         private void lexpush()
dvander@824:         {
dvander@824:             if (LexPushed)
dvander@824:             {
dvander@824:                 throw new ParseException("Lexer cannot be pushed twice");
dvander@824:             }
dvander@824:             LexPushed = true;
dvander@824:         }
dvander@824: 
dvander@824:         /* Attempts to match a string against the next stream tokens.
dvander@824:          * On success, the token stream is advanced.
dvander@824:          */
dvander@824:         private bool MatchString(int startPos, string str)
dvander@824:         {
dvander@824:             if (str.Length > Contents.Length - startPos)
dvander@824:             {
dvander@824:                 return false;
dvander@824:             }
dvander@824:             for (int i = 0; i < str.Length; i++)
dvander@824:             {
dvander@824:                 if (str[i] != Contents[startPos + i])
dvander@824:                 {
dvander@824:                     return false;
dvander@824:                 }
dvander@824:             }
dvander@824:             Contents.Remove(startPos, str.Length);
dvander@824:             return true;
dvander@824:         }
dvander@824: 
dvander@824:         /* Attempts to match a string against the next stream tokens */
dvander@824:         private bool CheckString(int startPos, string str)
dvander@824:         {
dvander@824:             if (str.Length > Contents.Length - startPos)
dvander@824:             {
dvander@824:                 return false;
dvander@824:             }
dvander@824:             for (int i = 0; i < str.Length; i++)
dvander@824:             {
dvander@824:                 if (str[i] != Contents[startPos + i])
dvander@824:                 {
dvander@824:                     return false;
dvander@824:                 }
dvander@824:             }
dvander@824:             return true;
dvander@824:         }
dvander@824: 
dvander@824:         /* Ignores a block starting and ending with { and } respectively */
dvander@824:         private void ignore_block()
dvander@824:         {
dvander@824:             /* Hrm, this is tricky.  First, expect an open brace. */
dvander@824:             NeedChar('{');
dvander@824:             /* Consume the entire function by keeping track of braces */
dvander@824:             int numBraces = 1;
dvander@824:             bool in_string = false;
dvander@824:             bool found_end = false;
dvander@824:             for (int i = 0; i < Contents.Length; i++)
dvander@824:             {
dvander@824:                 if (Contents[i] == '\n')
dvander@824:                 {
dvander@824:                     LineNo++;
dvander@824:                 }
dvander@824:                 if (!in_string)
dvander@824:                 {
dvander@824:                     if (Contents[i] == '"')
dvander@824:                     {
dvander@824:                         in_string = true;
dvander@824:                     }
dvander@824:                     else if (Contents[i] == '{')
dvander@824:                     {
dvander@824:                         numBraces++;
dvander@824:                     }
dvander@824:                     else if (Contents[i] == '}')
dvander@824:                     {
dvander@824:                         if (numBraces == 0)
dvander@824:                         {
dvander@824:                             throw new ParseException("Unable to find a matching open brace");
dvander@824:                         }
dvander@824:                         if (--numBraces == 0)
dvander@824:                         {
dvander@824:                             Contents.Remove(0, i + 1);
dvander@824:                             found_end = true;
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     }
dvander@824:                 }
dvander@824:                 else
dvander@824:                 {
dvander@824:                     /* It is possible to be in a string and at the first character */
dvander@824:                     if (Contents[i] == '"' && Contents[i - 1] != '\\')
dvander@824:                     {
dvander@824:                         in_string = false;
dvander@824:                     }
dvander@824:                 }
dvander@824:             }
dvander@824:             if (!found_end)
dvander@824:             {
dvander@824:                 throw new ParseException("Unable to find a matching close brace");
dvander@824:             }
dvander@824:         }
dvander@824: 
dvander@824:         /* Strips all non-doc C and C++ style comments. as well as unused pragmas */
dvander@824:         private void preprocess()
dvander@824:         {
dvander@824:             if (Contents.Length == 1)
dvander@824:             {
dvander@824:                 return;
dvander@824:             }
dvander@824: 
dvander@824:             bool in_comment = false;
dvander@824:             bool is_ml = false;
dvander@824:             bool is_doc = false;
dvander@824:             bool is_newline = true;
dvander@824: 
pred@2019:             for (int i=0; i<Contents.Length; i++)
dvander@824:             {
dvander@824:                 if (Contents[i] == '\n')
dvander@824:                 {
dvander@824:                     is_newline = true;
dvander@824:                 }
dvander@824:                 if (!in_comment)
dvander@824:                 {
dvander@824:                     if (Contents[i] == '/')
dvander@824:                     {
dvander@824:                         if (Contents[i+1] == '/')
dvander@824:                         {
dvander@824:                             in_comment = true;
dvander@824:                             is_ml = false;
dvander@824:                             is_doc = false;
pred@2019:                             is_newline = false;
dvander@824:                         } else if (Contents[i+1] == '*') {
dvander@824:                             /* Detect whether this is a doc comment or not */
dvander@824:                             if (Contents.Length - i >= 3
dvander@824:                                 && Contents[i+2] == '*')
dvander@824:                             {
dvander@824:                                 is_doc = true;
dvander@824:                             } else {
dvander@824:                                 is_doc = false;
dvander@824:                             }
dvander@824:                             in_comment = true;
dvander@824:                             is_ml = true;
dvander@824:                         }
dvander@824:                         if (in_comment && !is_doc)
dvander@824:                         {
dvander@824:                             Contents[i] = ' ';
dvander@824:                             Contents[i+1] = ' ';        /* Just for safety */
dvander@824:                         }
dvander@824:                     }
dvander@824:                     else if (is_newline)
dvander@824:                     {
dvander@824:                         if (Contents[i] == '#')
dvander@824:                         {
dvander@824:                             if (!CheckString(i, "#define")
dvander@824:                                 && !CheckString(i, "#include"))
dvander@824:                             {
dvander@824:                                 /* Ignore! */
dvander@824:                                 Contents[i] = ' ';
dvander@824:                                 in_comment = true;
dvander@824:                                 is_doc = false;
dvander@824:                                 is_ml = false;
dvander@824:                             }
dvander@824:                             is_newline = false;
dvander@824:                         }
dvander@824:                         else if (!Char.IsWhiteSpace(Contents[i]))
dvander@824:                         {
dvander@824:                             is_newline = false;
dvander@824:                         }
dvander@824:                     }
dvander@824:                 } else {
dvander@824:                     if (is_ml)
dvander@824:                     {
dvander@824:                         if (Contents[i] == '*' && Contents[i+1] == '/')
dvander@824:                         {
dvander@824:                             if (!is_doc)
dvander@824:                             {
dvander@824:                                 Contents[i] = ' ';
dvander@824:                                 Contents[i+1] = ' ';
dvander@824:                             }
dvander@824:                             in_comment = false;
dvander@824:                             i++;    /* Don't look at the next character */
dvander@824:                         }
dvander@824:                         else if (!is_doc && Contents[i] != '\n')
dvander@824:                         {
dvander@824:                             /* Erase comment */
dvander@824:                             Contents[i] = ' ';
dvander@824:                         }
dvander@824:                     } else {
dvander@824:                         if (is_newline)
dvander@824:                         {
dvander@824:                             in_comment = false;
dvander@824:                         }
dvander@824:                         else
dvander@824:                         {
dvander@824:                             /* Erase comment */
dvander@824:                             Contents[i] = ' ';
dvander@824:                         }
dvander@824:                     }
dvander@824:                 }
dvander@824:             }
dvander@824: 
dvander@824:             if (in_comment)
dvander@824:             {
dvander@824:                 throw new ParseException("Multi-line comment has no terminator");
dvander@824:             }
dvander@824:         }
dvander@824: 
dvander@824:         /* Strips whitespace */
dvander@824:         private void trim_whitespace_pre()
dvander@824:         {
dvander@824:             int spaces = 0;
dvander@824:             while (spaces < Contents.Length
dvander@824:                    && Contents[spaces] != '\0'
dvander@824:                    && Char.IsWhiteSpace(Contents[spaces]))
dvander@824:             {
dvander@824:                 if (Contents[spaces] == '\n')
dvander@824:                 {
dvander@824:                     LineNo++;
dvander@824:                 }
dvander@824:                 spaces++;
dvander@824:             }
dvander@824:             if (spaces > 0)
dvander@824:             {
dvander@824:                 Contents.Remove(0, spaces);
dvander@824:             }
dvander@824:         }
dvander@824: 
dvander@824:         private void preprocess_line()
dvander@824:         {
dvander@824:             trim_whitespace_pre();
dvander@824:         }
dvander@824: 
dvander@824:         private bool IsSymbolChar(char c, bool first)
dvander@824:         {
dvander@824:             bool valid = (first ? Char.IsLetter(c) : Char.IsLetterOrDigit(c));
dvander@824:             return (valid || (c == '_'));
dvander@824:         }
dvander@824: 
dvander@824:         private bool IsEscapedChar(char c)
dvander@824:         {
dvander@824:             return (c == 'n'
dvander@824:                     || c == 'r'
dvander@824:                     || c == 't'
dvander@824:                     || c == 'v'
dvander@824:                     || c == '\\'
dvander@824:                     || c == '"');
dvander@824:         }
dvander@824: 
dvander@824:         private bool IsNotKeywordChar(char c)
dvander@824:         {
dvander@824:             return (Char.IsWhiteSpace(c)
dvander@824:                     || (c == '-' || c == '+')
dvander@824:                     || (c == '/' || c == '*')
dvander@824:                     || (c == '(' || c == ')')
dvander@824:                     || (c == '=' || c == '%')
dvander@824:                     || (c == '>' || c == '<')
dvander@824:                     || (c == '!' || c == '~')
dvander@824:                     );
dvander@824:         }
dvander@824: 
dvander@824:         /* Consumes one lexical token and gathers information about it */
dvander@824:         private LexToken lex()
dvander@824:         {
dvander@824:             if (LexPushed)
dvander@824:             {
dvander@824:                 LexPushed = false;
dvander@824:                 return _LastToken;
dvander@824:             }
dvander@824: 
dvander@824:             /* Number of chars we will be deleting from the input */
dvander@824:             int stripchars = 0;
dvander@824: 
dvander@824:             /* Clear our state */
dvander@824:             _LastToken = LexToken.TOKEN_NONE;
dvander@824: 
dvander@824:             /* Remove stuff we don't want from the line */
dvander@824:             preprocess_line();
dvander@824: 
dvander@824:             if (Contents.Length < 1)
dvander@824:             {
dvander@824:                 _LastToken = LexToken.TOKEN_EOF;
dvander@824:                 return _LastToken;
dvander@824:             }
dvander@824:             
dvander@824:             /* Get the token list */
dvander@824:             KeywordToken[] tokens = Tokenizer.Tokens;
dvander@824:             for (int i = 0; i < tokens.Length; i++)
dvander@824:             {
dvander@824:                 if (!CheckString(0, tokens[i].keyword))
dvander@824:                 {
dvander@824:                     continue;
dvander@824:                 }
dvander@824:                 int len = tokens[i].keyword.Length;
dvander@824:                 /* Now check to see what the next token is */
dvander@824:                 if (Contents.Length == len
dvander@824:                     || IsNotKeywordChar(Contents[len]))
dvander@824:                 {
dvander@824:                     /* We have a token match! */
dvander@824:                     _LastToken = tokens[i].token;
dvander@824:                     _LexString = null;
dvander@824:                     _LexChar = '\0';
dvander@824:                     stripchars = len;
dvander@824:                     break;
dvander@824:                 }
dvander@824:             }
dvander@824: 
dvander@824:             if (_LastToken == LexToken.TOKEN_NONE)
dvander@824:             {
dvander@824:                 /* See if we can try to read a symbol */
dvander@824:                 if (IsSymbolChar(Contents[0], true))
dvander@824:                 {
dvander@824:                     int characters = 1;
dvander@824:                     while (Contents[characters] != '\0'
dvander@824:                            && IsSymbolChar(Contents[characters], false))
dvander@824:                     {
dvander@824:                         characters++;
dvander@824:                     }
dvander@824:                     stripchars = characters;
dvander@824:                     /* We're done! See what's next.. */
dvander@824:                     if (Contents[characters] == ':')
dvander@824:                     {
dvander@824:                         _LastToken = LexToken.TOKEN_LABEL;
dvander@824:                         stripchars++;
dvander@824:                     }
dvander@824:                     else
dvander@824:                     {
dvander@824:                         _LastToken = LexToken.TOKEN_SYMBOL;
dvander@824:                     }
dvander@824:                     _LexString = Contents.ToString().Substring(0, characters);
dvander@824:                     _LexChar = _LexString[0];
dvander@824:                 }/*
dvander@824:                 else if (Contents[0] == '"')
dvander@824:                 {
dvander@824:                     bool reached_end = false;
dvander@824:                     StringBuilder builder = new StringBuilder();
dvander@824:                     for (int i = 1; i < Contents.Length; i++)
dvander@824:                     {
dvander@824:                         if (i < Contents.Length - 1
dvander@824:                             && Contents[i] == '\\'
dvander@824:                             && IsEscapedChar(Contents[i+1]))
dvander@824:                         {
dvander@824:                             i++;
dvander@824:                         }
dvander@824:                         else if (Contents[i] == '"')
dvander@824:                         {
dvander@824:                             reached_end = true;
dvander@824:                             break;
dvander@824:                         }
dvander@824:                         builder.Append(Contents[i]);
dvander@824:                     }
dvander@824:                     if (!reached_end)
dvander@824:                     {
dvander@824:                         throw new ParseException("Expected end of string; none found");
dvander@824:                     }
dvander@824:                     _LexString = builder.ToString();
dvander@824:                     _LexChar = _LexString.Length > 0 ? _LexString[0] : '\0';
dvander@824:                 }*/
dvander@824:                 else if (Contents[0] == '\'')
dvander@824:                 {
dvander@824:                     char c = '\0';
dvander@824:                     if (Contents.Length < 3)
dvander@824:                     {
dvander@824:                         throw new ParseException("Expected end of character; none found");
dvander@824:                     }
dvander@824:                     else if (Contents[1] == '\\')
dvander@824:                     {
dvander@824:                         if (Contents.Length < 4 || !IsEscapedChar(Contents[2]))
dvander@824:                         {
dvander@824:                             throw new ParseException("Expected end of character; none found");
dvander@824:                         }
dvander@824:                         if (Contents[2] == 'r')
dvander@824:                         {
dvander@824:                             c = '\r';
dvander@824:                         }
dvander@824:                         else if (Contents[2] == 'n')
dvander@824:                         {
dvander@824:                             c = '\n';
dvander@824:                         }
dvander@824:                         else if (Contents[2] == 't')
dvander@824:                         {
dvander@824:                             c = '\t';
dvander@824:                         }
dvander@824:                         else if (Contents[2] == 'v')
dvander@824:                         {
dvander@824:                             c = '\v';
dvander@824:                         }
dvander@824:                         else if (Contents[2] == '\\')
dvander@824:                         {
dvander@824:                             c = '\\';
dvander@824:                         }
dvander@824:                         else if (Contents[2] == '"')
dvander@824:                         {
dvander@824:                             c = '"';
dvander@824:                         }
dvander@824:                         stripchars = 4;
dvander@824:                     }
dvander@824:                     else
dvander@824:                     {
dvander@824:                         c = Contents[1];
dvander@824:                         stripchars = 3;
dvander@824:                     }
dvander@824:                     _LexString = c.ToString();
dvander@824:                     _LexChar = c;
dvander@824:                     _LastToken = LexToken.TOKEN_QUOTCHAR;
dvander@824:                 }
dvander@824:                 else if (CheckString(0, "/**"))
dvander@824:                 {
dvander@824:                     int endpoint = 0;
dvander@824:                     for (int i=3; i<Contents.Length-1; i++)
dvander@824:                     {
dvander@824:                         if (Contents[i] == '\n')
dvander@824:                         {
dvander@824:                             LineNo++;
dvander@824:                         }
dvander@824:                         else if (Contents[i] == '*' && Contents[i+1] == '/')
dvander@824:                         {
dvander@824:                             endpoint = i+1;
dvander@824:                             break;
dvander@824:                         }
dvander@824:                     }
dvander@824:                     if (endpoint == 0)
dvander@824:                     {
dvander@824:                         throw new ParseException("Expected end of multi-line comment; found end of file");
dvander@824:                     }
dvander@824:                     /* We have to factor one more character in because we're zero based */
dvander@824:                     endpoint++;
dvander@824:                     _LastToken = LexToken.TOKEN_DOCBLOCK;
dvander@824:                     _LexString = Contents.ToString().Substring(0, endpoint);
dvander@824:                     _LexChar = _LexString[0];
dvander@824:                     stripchars = endpoint;
dvander@824:                 }
dvander@824:                 else
dvander@824:                 {
dvander@824:                     _LastToken = LexToken.TOKEN_CHARACTER;
dvander@824:                     _LexString = Contents[0].ToString();
dvander@824:                     _LexChar = Contents[0];
dvander@824:                     stripchars = 1;
dvander@824:                 }
dvander@824:             }
dvander@824: 
dvander@824:             /* Strip N chars */
dvander@824:             if (stripchars > 0)
dvander@824:             {
dvander@824:                 Contents.Remove(0, stripchars);
dvander@824:             }
dvander@824: 
dvander@824:             return _LastToken;
dvander@824:         }
dvander@824:     }
dvander@824: }