1.1 --- a/tools/incparser/IncParser.cs Fri Jul 11 06:51:21 2008 +0000
1.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000
1.3 @@ -1,1342 +0,0 @@
1.4 -using System;
1.5 -using System.Collections.Generic;
1.6 -using System.IO;
1.7 -using System.Text;
1.8 -
1.9 -namespace incparser
1.10 -{
1.11 - enum LexToken
1.12 - {
1.13 - TOKEN_NONE,
1.14 - TOKEN_INCLUDE,
1.15 - TOKEN_DEFINE,
1.16 - TOKEN_CONST,
1.17 - TOKEN_STRUCT,
1.18 - TOKEN_FUNCTAG,
1.19 - TOKEN_FUNCENUM,
1.20 - TOKEN_ENUM,
1.21 - TOKEN_NATIVE,
1.22 - TOKEN_STOCK,
1.23 - TOKEN_PUBLIC,
1.24 - TOKEN_FORWARD,
1.25 - TOKEN_QUOTCHAR,
1.26 - TOKEN_OPERATOR,
1.27 - TOKEN_CHARACTER,
1.28 - /*TOKEN_STRING,*/
1.29 - TOKEN_LABEL,
1.30 - TOKEN_SYMBOL,
1.31 - TOKEN_DOCBLOCK,
1.32 - TOKEN_EOF,
1.33 - }
1.34 -
1.35 - class ParseException : System.Exception
1.36 - {
1.37 - public ParseException(string message) : base(message)
1.38 - {
1.39 - }
1.40 - };
1.41 -
1.42 - class KeywordToken
1.43 - {
1.44 - public KeywordToken(string k, LexToken t)
1.45 - {
1.46 - keyword = k;
1.47 - token = t;
1.48 - }
1.49 - public string keyword;
1.50 - public LexToken token;
1.51 - };
1.52 -
1.53 - class Tokenizer
1.54 - {
1.55 - private static bool s_initialized = false;
1.56 - private static int s_position = 0;
1.57 - private static KeywordToken [] s_tokens = null;
1.58 - public static KeywordToken[] Tokens = null;
1.59 -
1.60 - public static void Initialize()
1.61 - {
1.62 - if (s_initialized)
1.63 - {
1.64 - return;
1.65 - }
1.66 - s_tokens = new KeywordToken[12];
1.67 - s_initialized = true;
1.68 - s_position = 0;
1.69 -
1.70 - AddToken("#include", LexToken.TOKEN_INCLUDE);
1.71 - AddToken("#define", LexToken.TOKEN_DEFINE);
1.72 - AddToken("native", LexToken.TOKEN_NATIVE);
1.73 - AddToken("functag", LexToken.TOKEN_FUNCTAG);
1.74 - AddToken("funcenum", LexToken.TOKEN_FUNCENUM);
1.75 - AddToken("enum", LexToken.TOKEN_ENUM);
1.76 - AddToken("stock", LexToken.TOKEN_STOCK);
1.77 - AddToken("public", LexToken.TOKEN_PUBLIC);
1.78 - AddToken("forward", LexToken.TOKEN_FORWARD);
1.79 - AddToken("const", LexToken.TOKEN_CONST);
1.80 - AddToken("struct", LexToken.TOKEN_STRUCT);
1.81 - AddToken("operator", LexToken.TOKEN_OPERATOR);
1.82 -
1.83 - Tokens = s_tokens;
1.84 - }
1.85 -
1.86 - private static void AddToken(string lex, LexToken tok)
1.87 - {
1.88 - s_tokens[s_position] = new KeywordToken(lex, tok);
1.89 - s_position++;
1.90 - }
1.91 - };
1.92 -
1.93 - class IncParser
1.94 - {
1.95 - private string FileName;
1.96 - private StringBuilder Contents;
1.97 - private bool LexPushed;
1.98 - private LexToken _LastToken;
1.99 - private string _LexString;
1.100 - private char _LexChar;
1.101 - private uint LineNo;
1.102 -
1.103 - public string LexString
1.104 - {
1.105 - get
1.106 - {
1.107 - return _LexString;
1.108 - }
1.109 - }
1.110 -
1.111 - public char LexChar
1.112 - {
1.113 - get
1.114 - {
1.115 - return _LexChar;
1.116 - }
1.117 - }
1.118 -
1.119 - public uint GetLineNumber()
1.120 - {
1.121 - return LineNo;
1.122 - }
1.123 -
1.124 - public IncParser(string file)
1.125 - {
1.126 - FileName = file;
1.127 -
1.128 - /* Clear out lexer stuff */
1.129 - LexPushed = false;
1.130 - _LastToken = LexToken.TOKEN_NONE;
1.131 - _LexString = null;
1.132 - _LexChar = '\0';
1.133 - LineNo = 1;
1.134 -
1.135 - /* Initialize; this can throw an exception */
1.136 - Initialize();
1.137 - }
1.138 -
1.139 - public void Parse(ParseWriter w)
1.140 - {
1.141 - w.BeginSection(System.IO.Path.GetFileName(FileName));
1.142 - LexToken tok = LexToken.TOKEN_NONE;
1.143 - while ((tok = lex()) != LexToken.TOKEN_EOF)
1.144 - {
1.145 - switch (tok)
1.146 - {
1.147 - case LexToken.TOKEN_DOCBLOCK:
1.148 - {
1.149 - w.WritePair("doc", LexString);
1.150 - break;
1.151 - }
1.152 - case LexToken.TOKEN_DEFINE:
1.153 - {
1.154 - PARSE_Define(w);
1.155 - break;
1.156 - }
1.157 - case LexToken.TOKEN_ENUM:
1.158 - {
1.159 - PARSE_Enum(w);
1.160 - break;
1.161 - }
1.162 - case LexToken.TOKEN_FORWARD:
1.163 - {
1.164 - PARSE_Function(tok, w);
1.165 - break;
1.166 - }
1.167 - case LexToken.TOKEN_NATIVE:
1.168 - {
1.169 - PARSE_Function(tok, w);
1.170 - break;
1.171 - }
1.172 - case LexToken.TOKEN_STOCK:
1.173 - {
1.174 - PARSE_Function(tok, w);
1.175 - break;
1.176 - }
1.177 - case LexToken.TOKEN_FUNCTAG:
1.178 - {
1.179 - PARSE_FuncTag(w);
1.180 - break;
1.181 - }
1.182 - case LexToken.TOKEN_FUNCENUM:
1.183 - {
1.184 - PARSE_FuncEnum(w);
1.185 - break;
1.186 - }
1.187 - case LexToken.TOKEN_INCLUDE:
1.188 - {
1.189 - PARSE_Include(w);
1.190 - break;
1.191 - }
1.192 - case LexToken.TOKEN_STRUCT:
1.193 - {
1.194 - PARSE_Struct(w);
1.195 - break;
1.196 - }
1.197 - case LexToken.TOKEN_PUBLIC:
1.198 - {
1.199 - PARSE_Public(w);
1.200 - break;
1.201 - }
1.202 - default:
1.203 - {
1.204 - if (_LexString == "static" || _LexString == "new")
1.205 - {
1.206 - PARSE_Public(w);
1.207 - break;
1.208 - }
1.209 - throw new ParseException("Unrecognized token: " + tok + " " + _LexString);
1.210 - }
1.211 - }
1.212 - }
1.213 - w.EndSection();
1.214 - }
1.215 -
1.216 - private void PARSE_Public(ParseWriter w)
1.217 - {
1.218 - MatchToken(LexToken.TOKEN_CONST);
1.219 -
1.220 - /* Eat up an optional tag */
1.221 - MatchToken(LexToken.TOKEN_LABEL);
1.222 -
1.223 - /* Eat up a name */
1.224 - NeedToken(LexToken.TOKEN_SYMBOL);
1.225 -
1.226 - while (MatchChar('['))
1.227 - {
1.228 - DiscardUntilChar(']');
1.229 - NeedChar(']');
1.230 - }
1.231 -
1.232 - if (MatchChar('='))
1.233 - {
1.234 - ignore_block();
1.235 - }
1.236 - else if (MatchChar('(') && MatchChar(')'))
1.237 - {
1.238 - ignore_block();
1.239 - return;
1.240 - }
1.241 -
1.242 - NeedChar(';');
1.243 - }
1.244 -
1.245 - private void PARSE_Struct(ParseWriter w)
1.246 - {
1.247 - /* For now, we completely ignore these (they're not written to the output) */
1.248 - NeedToken(LexToken.TOKEN_SYMBOL);
1.249 - w.structList.Add(LexString);
1.250 - NeedChar('{');
1.251 -
1.252 - bool need_closebrace = true;
1.253 - bool more_entries = true;
1.254 - do
1.255 - {
1.256 - if (MatchChar('}'))
1.257 - {
1.258 - need_closebrace = false;
1.259 - break;
1.260 - }
1.261 - MatchToken(LexToken.TOKEN_CONST);
1.262 - MatchToken(LexToken.TOKEN_LABEL);
1.263 - NeedToken(LexToken.TOKEN_SYMBOL);
1.264 - while (MatchChar('['))
1.265 - {
1.266 - DiscardUntilChar(']');
1.267 - NeedChar(']');
1.268 - }
1.269 - more_entries = MatchChar(',');
1.270 - MatchToken(LexToken.TOKEN_DOCBLOCK);
1.271 - if (!more_entries)
1.272 - {
1.273 - break;
1.274 - }
1.275 - } while (true);
1.276 - if (need_closebrace)
1.277 - {
1.278 - NeedChar('}');
1.279 - }
1.280 - NeedChar(';');
1.281 - }
1.282 -
1.283 - private void PARSE_Include(ParseWriter w)
1.284 - {
1.285 - /* read until the end of the line */
1.286 - int index = Contents.ToString().IndexOf('\n');
1.287 - string value;
1.288 - if (index == -1)
1.289 - {
1.290 - value = Contents.ToString();
1.291 - Contents.Remove(0, value.Length);
1.292 - }
1.293 - else if (index == 0)
1.294 - {
1.295 - value = "";
1.296 - }
1.297 - else
1.298 - {
1.299 - value = Contents.ToString().Substring(0, index);
1.300 - Contents.Remove(0, index);
1.301 - }
1.302 -
1.303 - /* Strip whitespace */
1.304 - value = value.Trim('\r', '\t', ' ');
1.305 -
1.306 - /* Write */
1.307 - w.BeginSection("include");
1.308 - w.WritePair("name", value);
1.309 - w.EndSection();
1.310 - }
1.311 -
1.312 - private void PARSE_FuncEnum(ParseWriter w)
1.313 - {
1.314 - w.BeginSection("funcenum");
1.315 -
1.316 - /* Get the functag name */
1.317 - NeedToken(LexToken.TOKEN_SYMBOL);
1.318 - w.WritePair("name", LexString);
1.319 - w.funcenumList.Add(LexString);
1.320 -
1.321 - NeedChar('{');
1.322 -
1.323 - bool need_closebrace = true;
1.324 - do
1.325 - {
1.326 - /* Shortcut out? */
1.327 - if (MatchChar('}'))
1.328 - {
1.329 - need_closebrace = false;
1.330 - break;
1.331 - }
1.332 -
1.333 - /* Start function section */
1.334 - w.BeginSection("function");
1.335 -
1.336 - if (MatchToken(LexToken.TOKEN_DOCBLOCK))
1.337 - {
1.338 - w.WritePair("doc", LexString);
1.339 - }
1.340 -
1.341 - /* Get the return tag */
1.342 - if (MatchToken(LexToken.TOKEN_LABEL))
1.343 - {
1.344 - w.WritePair("return", LexString);
1.345 - }
1.346 -
1.347 - /* Get function type */
1.348 - if (MatchToken(LexToken.TOKEN_PUBLIC))
1.349 - {
1.350 - w.WritePair("type", "public");
1.351 - }
1.352 - else if (MatchToken(LexToken.TOKEN_STOCK))
1.353 - {
1.354 - w.WritePair("type", "stock");
1.355 - }
1.356 -
1.357 - /* Parse the parameters */
1.358 - ParseParameters(w);
1.359 -
1.360 - /* End the section */
1.361 - w.EndSection();
1.362 -
1.363 - if (!MatchChar(','))
1.364 - {
1.365 - break;
1.366 - }
1.367 - } while (true);
1.368 -
1.369 - if (need_closebrace)
1.370 - {
1.371 - NeedChar('}');
1.372 - }
1.373 - NeedChar(';');
1.374 -
1.375 - w.EndSection();
1.376 - }
1.377 -
1.378 - private void PARSE_FuncTag(ParseWriter w)
1.379 - {
1.380 - w.BeginSection("functag");
1.381 -
1.382 - /* Get the functag name */
1.383 - NeedToken(LexToken.TOKEN_SYMBOL);
1.384 - w.WritePair("name", LexString);
1.385 - w.functagList.Add(LexString);
1.386 -
1.387 - if (MatchToken(LexToken.TOKEN_LABEL))
1.388 - {
1.389 - w.WritePair("return", LexString);
1.390 - }
1.391 -
1.392 - if (MatchToken(LexToken.TOKEN_PUBLIC))
1.393 - {
1.394 - w.WritePair("type", "public");
1.395 - }
1.396 - else if (MatchToken(LexToken.TOKEN_STOCK))
1.397 - {
1.398 - w.WritePair("type", "stock");
1.399 - }
1.400 -
1.401 - ParseParameters(w);
1.402 - NeedChar(';');
1.403 -
1.404 - w.EndSection();
1.405 - }
1.406 -
1.407 - private void PARSE_Function(LexToken tok, ParseWriter w)
1.408 - {
1.409 - string tag="", name;
1.410 -
1.411 - /* Get the return value */
1.412 - if (MatchToken(LexToken.TOKEN_LABEL))
1.413 - {
1.414 - tag = LexString;
1.415 - }
1.416 -
1.417 - if (MatchToken(LexToken.TOKEN_OPERATOR))
1.418 - {
1.419 - /* Check if we're some sort of invalid function */
1.420 - DiscardUntilCharOrComment('(', false);
1.421 - ParseParameters(null);
1.422 - if (tok == LexToken.TOKEN_STOCK)
1.423 - {
1.424 - ignore_block();
1.425 - }
1.426 - else if (tok == LexToken.TOKEN_NATIVE || tok == LexToken.TOKEN_FORWARD)
1.427 - {
1.428 - if (MatchChar('='))
1.429 - {
1.430 - DiscardUntilCharOrComment(';', false);
1.431 - }
1.432 - NeedChar(';');
1.433 - }
1.434 - return;
1.435 - }
1.436 -
1.437 - /* Get the name */
1.438 - NeedToken(LexToken.TOKEN_SYMBOL);
1.439 - name = LexString;
1.440 -
1.441 - PARSE_Function(tok, w, tag, name);
1.442 - }
1.443 -
1.444 - private void PARSE_Function(LexToken tok, ParseWriter w, string tag, string name)
1.445 - {
1.446 - if (tok == LexToken.TOKEN_FORWARD)
1.447 - {
1.448 - w.BeginSection("forward");
1.449 - w.forwardList.Add(name);
1.450 - }
1.451 - else if (tok == LexToken.TOKEN_NATIVE)
1.452 - {
1.453 - w.BeginSection("native");
1.454 - w.nativeList.Add(name);
1.455 - }
1.456 - else if (tok == LexToken.TOKEN_STOCK)
1.457 - {
1.458 - w.BeginSection("stock");
1.459 - w.stockList.Add(name);
1.460 - }
1.461 -
1.462 - w.WritePair("name", name);
1.463 -
1.464 - if (tag.Length > 0)
1.465 - {
1.466 - w.WritePair("return", tag);
1.467 - }
1.468 -
1.469 - ParseParameters(w);
1.470 -
1.471 - if (tok == LexToken.TOKEN_STOCK)
1.472 - {
1.473 - ignore_block();
1.474 - }
1.475 - else
1.476 - {
1.477 - /* Make sure there's a semicolon */
1.478 - if (MatchChar('='))
1.479 - {
1.480 - DiscardUntilCharOrComment(';', false);
1.481 - }
1.482 - NeedChar(';');
1.483 - }
1.484 -
1.485 - w.EndSection();
1.486 - }
1.487 -
1.488 - private void ParseParameters(ParseWriter w)
1.489 - {
1.490 - NeedChar('(');
1.491 - string extra;
1.492 - do
1.493 - {
1.494 - if (MatchChar(')'))
1.495 - {
1.496 - return;
1.497 - }
1.498 -
1.499 - /* Start the parameter */
1.500 - if (w != null)
1.501 - {
1.502 - w.BeginSection("parameter");
1.503 - }
1.504 -
1.505 - /* Check for a 'const' token */
1.506 - if (MatchToken(LexToken.TOKEN_CONST) && w != null)
1.507 - {
1.508 - w.WritePair("const", "true");
1.509 - }
1.510 -
1.511 - /* Check if it's by reference */
1.512 - if (MatchChar('&') && w != null)
1.513 - {
1.514 - w.WritePair("byref", "true");
1.515 - }
1.516 -
1.517 - /* Check for a tag */
1.518 - if (MatchToken(LexToken.TOKEN_LABEL) && w != null)
1.519 - {
1.520 - w.WritePair("tag", LexString);
1.521 - }
1.522 -
1.523 - /* Get the symbol and write it */
1.524 - if (MatchString(0, "...") && w != null)
1.525 - {
1.526 - w.WritePair("name", "...");
1.527 - }
1.528 - else
1.529 - {
1.530 - NeedToken(LexToken.TOKEN_SYMBOL);
1.531 - if (w != null)
1.532 - {
1.533 - w.WritePair("name", LexString);
1.534 - }
1.535 - }
1.536 -
1.537 - /* Check if we have a default value yet */
1.538 - bool default_value = false;
1.539 - if (MatchChar('='))
1.540 - {
1.541 - default_value = true;
1.542 - }
1.543 - else if (MatchChar('['))
1.544 - {
1.545 - extra = "";
1.546 - string temp;
1.547 - do
1.548 - {
1.549 - extra += "[";
1.550 - temp = ConsumeUntilChar(']');
1.551 - if (temp != null)
1.552 - {
1.553 - extra += temp;
1.554 - }
1.555 - extra += "]";
1.556 - NeedChar(']');
1.557 - } while (MatchChar('['));
1.558 - w.WritePair("decoration", extra);
1.559 - }
1.560 -
1.561 - /* If we have a default value, get it */
1.562 - if (default_value || MatchChar('='))
1.563 - {
1.564 - if ((extra = ConsumeParamDefValue()) == null)
1.565 - {
1.566 - throw new ParseException("Expected a default value; found none");
1.567 - }
1.568 - if (w != null)
1.569 - {
1.570 - w.WritePair("defval", extra);
1.571 - }
1.572 - }
1.573 -
1.574 - /* End the parameter */
1.575 - if (w != null)
1.576 - {
1.577 - w.EndSection();
1.578 - }
1.579 -
1.580 - if (!MatchChar(','))
1.581 - {
1.582 - break;
1.583 - }
1.584 - } while (true);
1.585 -
1.586 - NeedChar(')');
1.587 - }
1.588 -
1.589 - private void PARSE_Enum(ParseWriter w)
1.590 - {
1.591 - string name = "";
1.592 - if (MatchToken(LexToken.TOKEN_SYMBOL))
1.593 - {
1.594 - name = LexString;
1.595 - }
1.596 -
1.597 - w.BeginSection("enum");
1.598 - w.WritePair("name", name);
1.599 - w.BeginSection("properties");
1.600 -
1.601 - w.enumList.Add(name);
1.602 -
1.603 - NeedChar('{');
1.604 - bool more_entries = true;
1.605 - bool need_closebrace = true;
1.606 -
1.607 - if (MatchToken(LexToken.TOKEN_DOCBLOCK))
1.608 - {
1.609 - w.WritePair("doc", LexString);
1.610 - }
1.611 -
1.612 - do
1.613 - {
1.614 - if (MatchChar('}'))
1.615 - {
1.616 - need_closebrace = false;
1.617 - break;
1.618 - }
1.619 - NeedToken(LexToken.TOKEN_SYMBOL);
1.620 - name = LexString;
1.621 -
1.622 - if (MatchChar('='))
1.623 - {
1.624 - DiscardUntilCharOrComment(",\n}", true);
1.625 - }
1.626 - more_entries = MatchChar(',');
1.627 - w.WritePair("name", name);
1.628 - w.enumTypeList.Add(name);
1.629 - if (MatchToken(LexToken.TOKEN_DOCBLOCK))
1.630 - {
1.631 - w.WritePair("doc", LexString);
1.632 - }
1.633 - } while (more_entries);
1.634 - if (need_closebrace)
1.635 - {
1.636 - NeedChar('}');
1.637 - }
1.638 - NeedChar(';');
1.639 - w.EndSection();
1.640 - w.EndSection();
1.641 - }
1.642 -
1.643 - private void PARSE_Define(ParseWriter w)
1.644 - {
1.645 - /* first, get the symbol name */
1.646 - NeedToken(LexToken.TOKEN_SYMBOL);
1.647 - string name = LexString;
1.648 - string value;
1.649 -
1.650 - /* read until we hit the end of a line OR a comment */
1.651 - int endpoint = -1;
1.652 - for (int i = 0; i < Contents.Length; i++)
1.653 - {
1.654 - if (Contents[i] == '\n')
1.655 - {
1.656 - endpoint = i;
1.657 - break;
1.658 - }
1.659 - else if (CheckString(i, "/**"))
1.660 - {
1.661 - endpoint = i;
1.662 - break;
1.663 - }
1.664 - }
1.665 -
1.666 - if (endpoint < 1)
1.667 - {
1.668 - value = "";
1.669 - }
1.670 - else
1.671 - {
1.672 - value = Contents.ToString().Substring(0, endpoint);
1.673 - Contents.Remove(0, endpoint);
1.674 - }
1.675 -
1.676 - /* Write */
1.677 - w.defineList.Add(name);
1.678 -
1.679 - w.BeginSection("define");
1.680 - w.WritePair("name", name);
1.681 - w.WritePair("linetext", value);
1.682 - w.EndSection();
1.683 - }
1.684 -
1.685 - /* Reads up to a character and discards all text before it.
1.686 - */
1.687 - private void DiscardUntilChar(char c)
1.688 - {
1.689 - int eol = Contents.ToString().IndexOf(c);
1.690 -
1.691 - if (eol < 1)
1.692 - {
1.693 - return;
1.694 - }
1.695 -
1.696 - Contents.Remove(0, eol);
1.697 - }
1.698 -
1.699 - /* Same as DiscardUntilChar, except the discarded
1.700 - * text is returned first.
1.701 - */
1.702 - private string ConsumeUntilChar(char c)
1.703 - {
1.704 - int eol = Contents.ToString().IndexOf(c);
1.705 -
1.706 - if (eol < 1)
1.707 - {
1.708 - return null;
1.709 - }
1.710 -
1.711 - string rest = Contents.ToString().Substring(0, eol);
1.712 - Contents.Remove(0, eol);
1.713 - return rest;
1.714 - }
1.715 -
1.716 - /**
1.717 - * Returns a parameter's default value.
1.718 - */
1.719 - private string ConsumeParamDefValue()
1.720 - {
1.721 - int eol = FindEndOfExprLine(0);
1.722 - if (eol < 1)
1.723 - {
1.724 - return null;
1.725 - }
1.726 -
1.727 - string rest = Contents.ToString().Substring(0, eol);
1.728 - Contents.Remove(0, eol);
1.729 - return rest;
1.730 - }
1.731 -
1.732 - /**
1.733 - * Discards all data up to a certain character, ignoring
1.734 - * the contents, unless specified to stop at a doc block comment.
1.735 - */
1.736 - private void DiscardUntilCharOrComment(char c, bool stop_at_comment)
1.737 - {
1.738 - for (int i = 0; i < Contents.Length; i++)
1.739 - {
1.740 - if (Contents[i] == '\n')
1.741 - {
1.742 - LineNo++;
1.743 - }
1.744 - else if (Contents[i] == c)
1.745 - {
1.746 - if (i > 0)
1.747 - {
1.748 - Contents.Remove(0, i);
1.749 - }
1.750 - return;
1.751 - }
1.752 - else if (stop_at_comment && CheckString(i, "/**"))
1.753 - {
1.754 - if (i > 0)
1.755 - {
1.756 - Contents.Remove(0, i);
1.757 - }
1.758 - return;
1.759 - }
1.760 - }
1.761 - }
1.762 -
1.763 - /**
1.764 - * Discards all data up to a certain character, ignoring
1.765 - * the contents, unless specified to stop at a doc block comment.
1.766 - */
1.767 - private void DiscardUntilCharOrComment(string s, bool stop_at_comment)
1.768 - {
1.769 - for (int i = 0; i < Contents.Length; i++)
1.770 - {
1.771 - if (Contents[i] == '\n')
1.772 - {
1.773 - LineNo++;
1.774 - }
1.775 - else if (s.Contains(Contents[i].ToString()))
1.776 - {
1.777 - if (i > 0)
1.778 - {
1.779 - Contents.Remove(0, i);
1.780 - }
1.781 - return;
1.782 - }
1.783 - else if (stop_at_comment && CheckString(i, "/**"))
1.784 - {
1.785 - if (i > 0)
1.786 - {
1.787 - Contents.Remove(0, i);
1.788 - }
1.789 - return;
1.790 - }
1.791 - }
1.792 - }
1.793 -
1.794 - /**
1.795 - * Finds the end of an expression
1.796 - */
1.797 - private int FindEndOfExprLine(int start)
1.798 - {
1.799 - for (int i = start; i < Contents.Length; i++)
1.800 - {
1.801 - if (Contents[i] == '{')
1.802 - {
1.803 - if ((i = FindEndOfConsumptionLine('}', start+1)) == -1)
1.804 - {
1.805 - throw new ParseException("Expected token: } (found none)");
1.806 - }
1.807 - }
1.808 - else if (Contents[i] == ')' || Contents[i] == ',')
1.809 - {
1.810 - return i;
1.811 - }
1.812 - }
1.813 -
1.814 - return -1;
1.815 - }
1.816 -
1.817 - private int FindEndOfConsumptionLine(char c, int start)
1.818 - {
1.819 - for (int i = start; i < Contents.Length; i++)
1.820 - {
1.821 - if (Contents[i] == '{')
1.822 - {
1.823 - if ((i = FindEndOfConsumptionLine('}', i + 1)) == -1)
1.824 - {
1.825 - throw new ParseException("Expected token: } (found none)");
1.826 - }
1.827 - }
1.828 - else if (Contents[i] == c)
1.829 - {
1.830 - return i;
1.831 - }
1.832 - }
1.833 -
1.834 - return -1;
1.835 - }
1.836 -
1.837 - private void Initialize()
1.838 - {
1.839 - /* Open file, read it, and close it */
1.840 - StreamReader sr = File.OpenText(FileName);
1.841 - Contents = new StringBuilder(sr.ReadToEnd());
1.842 - sr.Close();
1.843 -
1.844 - /* Strip comments (may throw an exception) */
1.845 - preprocess();
1.846 -
1.847 - /* Lastly, initialize the tokenizer */
1.848 - Tokenizer.Initialize();
1.849 - }
1.850 -
1.851 - /* Consumes a token iff it matches */
1.852 - private bool MatchToken(LexToken token)
1.853 - {
1.854 - LexToken tok = lex();
1.855 - if (tok != token)
1.856 - {
1.857 - lexpush();
1.858 - return false;
1.859 - }
1.860 - return true;
1.861 - }
1.862 -
1.863 - /* Consumes a token and aborts the parser if there is no match */
1.864 - private void NeedToken(LexToken token)
1.865 - {
1.866 - if (!MatchToken(token))
1.867 - {
1.868 - throw new ParseException("Expected token: " + token);
1.869 - }
1.870 - }
1.871 -
1.872 - /* Consumes a character iff it matches */
1.873 - private bool MatchChar(char c)
1.874 - {
1.875 - if (lex() != LexToken.TOKEN_CHARACTER)
1.876 - {
1.877 - lexpush();
1.878 - return false;
1.879 - }
1.880 - if (_LexChar != c)
1.881 - {
1.882 - lexpush();
1.883 - return false;
1.884 - }
1.885 - return true;
1.886 - }
1.887 -
1.888 - /* Consumes a character or aborts the parser if there is no match */
1.889 - private void NeedChar(char c)
1.890 - {
1.891 - if (!MatchChar(c))
1.892 - {
1.893 - throw new ParseException("Expected character: " + c);
1.894 - }
1.895 - }
1.896 -
1.897 - /* Saves the lexer state so we don't reparse unnecessarily */
1.898 - private void lexpush()
1.899 - {
1.900 - if (LexPushed)
1.901 - {
1.902 - throw new ParseException("Lexer cannot be pushed twice");
1.903 - }
1.904 - LexPushed = true;
1.905 - }
1.906 -
1.907 - /* Attempts to match a string against the next stream tokens.
1.908 - * On success, the token stream is advanced.
1.909 - */
1.910 - private bool MatchString(int startPos, string str)
1.911 - {
1.912 - if (str.Length > Contents.Length - startPos)
1.913 - {
1.914 - return false;
1.915 - }
1.916 - for (int i = 0; i < str.Length; i++)
1.917 - {
1.918 - if (str[i] != Contents[startPos + i])
1.919 - {
1.920 - return false;
1.921 - }
1.922 - }
1.923 - Contents.Remove(startPos, str.Length);
1.924 - return true;
1.925 - }
1.926 -
1.927 - /* Attempts to match a string against the next stream tokens */
1.928 - private bool CheckString(int startPos, string str)
1.929 - {
1.930 - if (str.Length > Contents.Length - startPos)
1.931 - {
1.932 - return false;
1.933 - }
1.934 - for (int i = 0; i < str.Length; i++)
1.935 - {
1.936 - if (str[i] != Contents[startPos + i])
1.937 - {
1.938 - return false;
1.939 - }
1.940 - }
1.941 - return true;
1.942 - }
1.943 -
1.944 - /* Ignores a block starting and ending with { and } respectively */
1.945 - private void ignore_block()
1.946 - {
1.947 - /* Hrm, this is tricky. First, expect an open brace. */
1.948 - NeedChar('{');
1.949 - /* Consume the entire function by keeping track of braces */
1.950 - int numBraces = 1;
1.951 - bool in_string = false;
1.952 - bool found_end = false;
1.953 - for (int i = 0; i < Contents.Length; i++)
1.954 - {
1.955 - if (Contents[i] == '\n')
1.956 - {
1.957 - LineNo++;
1.958 - }
1.959 - if (!in_string)
1.960 - {
1.961 - if (Contents[i] == '"')
1.962 - {
1.963 - in_string = true;
1.964 - }
1.965 - else if (Contents[i] == '{')
1.966 - {
1.967 - numBraces++;
1.968 - }
1.969 - else if (Contents[i] == '}')
1.970 - {
1.971 - if (numBraces == 0)
1.972 - {
1.973 - throw new ParseException("Unable to find a matching open brace");
1.974 - }
1.975 - if (--numBraces == 0)
1.976 - {
1.977 - Contents.Remove(0, i + 1);
1.978 - found_end = true;
1.979 - break;
1.980 - }
1.981 - }
1.982 - }
1.983 - else
1.984 - {
1.985 - /* It is possible to be in a string and at the first character */
1.986 - if (Contents[i] == '"' && Contents[i - 1] != '\\')
1.987 - {
1.988 - in_string = false;
1.989 - }
1.990 - }
1.991 - }
1.992 - if (!found_end)
1.993 - {
1.994 - throw new ParseException("Unable to find a matching close brace");
1.995 - }
1.996 - }
1.997 -
1.998 - /* Strips all non-doc C and C++ style comments. as well as unused pragmas */
1.999 - private void preprocess()
1.1000 - {
1.1001 - if (Contents.Length == 1)
1.1002 - {
1.1003 - return;
1.1004 - }
1.1005 -
1.1006 - bool in_comment = false;
1.1007 - bool is_ml = false;
1.1008 - bool is_doc = false;
1.1009 - bool is_newline = true;
1.1010 -
1.1011 - for (int i=0; i<Contents.Length; i++)
1.1012 - {
1.1013 - if (Contents[i] == '\n')
1.1014 - {
1.1015 - is_newline = true;
1.1016 - }
1.1017 - if (!in_comment)
1.1018 - {
1.1019 - if (Contents[i] == '/')
1.1020 - {
1.1021 - if (Contents[i+1] == '/')
1.1022 - {
1.1023 - in_comment = true;
1.1024 - is_ml = false;
1.1025 - is_doc = false;
1.1026 - is_newline = false;
1.1027 - } else if (Contents[i+1] == '*') {
1.1028 - /* Detect whether this is a doc comment or not */
1.1029 - if (Contents.Length - i >= 3
1.1030 - && Contents[i+2] == '*')
1.1031 - {
1.1032 - is_doc = true;
1.1033 - } else {
1.1034 - is_doc = false;
1.1035 - }
1.1036 - in_comment = true;
1.1037 - is_ml = true;
1.1038 - }
1.1039 - if (in_comment && !is_doc)
1.1040 - {
1.1041 - Contents[i] = ' ';
1.1042 - Contents[i+1] = ' '; /* Just for safety */
1.1043 - }
1.1044 - }
1.1045 - else if (is_newline)
1.1046 - {
1.1047 - if (Contents[i] == '#')
1.1048 - {
1.1049 - if (!CheckString(i, "#define")
1.1050 - && !CheckString(i, "#include"))
1.1051 - {
1.1052 - /* Ignore! */
1.1053 - Contents[i] = ' ';
1.1054 - in_comment = true;
1.1055 - is_doc = false;
1.1056 - is_ml = false;
1.1057 - }
1.1058 - is_newline = false;
1.1059 - }
1.1060 - else if (!Char.IsWhiteSpace(Contents[i]))
1.1061 - {
1.1062 - is_newline = false;
1.1063 - }
1.1064 - }
1.1065 - } else {
1.1066 - if (is_ml)
1.1067 - {
1.1068 - if (Contents[i] == '*' && Contents[i+1] == '/')
1.1069 - {
1.1070 - if (!is_doc)
1.1071 - {
1.1072 - Contents[i] = ' ';
1.1073 - Contents[i+1] = ' ';
1.1074 - }
1.1075 - in_comment = false;
1.1076 - i++; /* Don't look at the next character */
1.1077 - }
1.1078 - else if (!is_doc && Contents[i] != '\n')
1.1079 - {
1.1080 - /* Erase comment */
1.1081 - Contents[i] = ' ';
1.1082 - }
1.1083 - } else {
1.1084 - if (is_newline)
1.1085 - {
1.1086 - in_comment = false;
1.1087 - }
1.1088 - else
1.1089 - {
1.1090 - /* Erase comment */
1.1091 - Contents[i] = ' ';
1.1092 - }
1.1093 - }
1.1094 - }
1.1095 - }
1.1096 -
1.1097 - if (in_comment)
1.1098 - {
1.1099 - throw new ParseException("Multi-line comment has no terminator");
1.1100 - }
1.1101 - }
1.1102 -
1.1103 - /* Strips whitespace */
1.1104 - private void trim_whitespace_pre()
1.1105 - {
1.1106 - int spaces = 0;
1.1107 - while (spaces < Contents.Length
1.1108 - && Contents[spaces] != '\0'
1.1109 - && Char.IsWhiteSpace(Contents[spaces]))
1.1110 - {
1.1111 - if (Contents[spaces] == '\n')
1.1112 - {
1.1113 - LineNo++;
1.1114 - }
1.1115 - spaces++;
1.1116 - }
1.1117 - if (spaces > 0)
1.1118 - {
1.1119 - Contents.Remove(0, spaces);
1.1120 - }
1.1121 - }
1.1122 -
1.1123 - private void preprocess_line()
1.1124 - {
1.1125 - trim_whitespace_pre();
1.1126 - }
1.1127 -
1.1128 - private bool IsSymbolChar(char c, bool first)
1.1129 - {
1.1130 - bool valid = (first ? Char.IsLetter(c) : Char.IsLetterOrDigit(c));
1.1131 - return (valid || (c == '_'));
1.1132 - }
1.1133 -
1.1134 - private bool IsEscapedChar(char c)
1.1135 - {
1.1136 - return (c == 'n'
1.1137 - || c == 'r'
1.1138 - || c == 't'
1.1139 - || c == 'v'
1.1140 - || c == '\\'
1.1141 - || c == '"');
1.1142 - }
1.1143 -
1.1144 - private bool IsNotKeywordChar(char c)
1.1145 - {
1.1146 - return (Char.IsWhiteSpace(c)
1.1147 - || (c == '-' || c == '+')
1.1148 - || (c == '/' || c == '*')
1.1149 - || (c == '(' || c == ')')
1.1150 - || (c == '=' || c == '%')
1.1151 - || (c == '>' || c == '<')
1.1152 - || (c == '!' || c == '~')
1.1153 - );
1.1154 - }
1.1155 -
1.1156 - /* Consumes one lexical token and gathers information about it */
1.1157 - private LexToken lex()
1.1158 - {
1.1159 - if (LexPushed)
1.1160 - {
1.1161 - LexPushed = false;
1.1162 - return _LastToken;
1.1163 - }
1.1164 -
1.1165 - /* Number of chars we will be deleting from the input */
1.1166 - int stripchars = 0;
1.1167 -
1.1168 - /* Clear our state */
1.1169 - _LastToken = LexToken.TOKEN_NONE;
1.1170 -
1.1171 - /* Remove stuff we don't want from the line */
1.1172 - preprocess_line();
1.1173 -
1.1174 - if (Contents.Length < 1)
1.1175 - {
1.1176 - _LastToken = LexToken.TOKEN_EOF;
1.1177 - return _LastToken;
1.1178 - }
1.1179 -
1.1180 - /* Get the token list */
1.1181 - KeywordToken[] tokens = Tokenizer.Tokens;
1.1182 - for (int i = 0; i < tokens.Length; i++)
1.1183 - {
1.1184 - if (!CheckString(0, tokens[i].keyword))
1.1185 - {
1.1186 - continue;
1.1187 - }
1.1188 - int len = tokens[i].keyword.Length;
1.1189 - /* Now check to see what the next token is */
1.1190 - if (Contents.Length == len
1.1191 - || IsNotKeywordChar(Contents[len]))
1.1192 - {
1.1193 - /* We have a token match! */
1.1194 - _LastToken = tokens[i].token;
1.1195 - _LexString = null;
1.1196 - _LexChar = '\0';
1.1197 - stripchars = len;
1.1198 - break;
1.1199 - }
1.1200 - }
1.1201 -
1.1202 - if (_LastToken == LexToken.TOKEN_NONE)
1.1203 - {
1.1204 - /* See if we can try to read a symbol */
1.1205 - if (IsSymbolChar(Contents[0], true))
1.1206 - {
1.1207 - int characters = 1;
1.1208 - while (Contents[characters] != '\0'
1.1209 - && IsSymbolChar(Contents[characters], false))
1.1210 - {
1.1211 - characters++;
1.1212 - }
1.1213 - stripchars = characters;
1.1214 - /* We're done! See what's next.. */
1.1215 - if (Contents[characters] == ':')
1.1216 - {
1.1217 - _LastToken = LexToken.TOKEN_LABEL;
1.1218 - stripchars++;
1.1219 - }
1.1220 - else
1.1221 - {
1.1222 - _LastToken = LexToken.TOKEN_SYMBOL;
1.1223 - }
1.1224 - _LexString = Contents.ToString().Substring(0, characters);
1.1225 - _LexChar = _LexString[0];
1.1226 - }/*
1.1227 - else if (Contents[0] == '"')
1.1228 - {
1.1229 - bool reached_end = false;
1.1230 - StringBuilder builder = new StringBuilder();
1.1231 - for (int i = 1; i < Contents.Length; i++)
1.1232 - {
1.1233 - if (i < Contents.Length - 1
1.1234 - && Contents[i] == '\\'
1.1235 - && IsEscapedChar(Contents[i+1]))
1.1236 - {
1.1237 - i++;
1.1238 - }
1.1239 - else if (Contents[i] == '"')
1.1240 - {
1.1241 - reached_end = true;
1.1242 - break;
1.1243 - }
1.1244 - builder.Append(Contents[i]);
1.1245 - }
1.1246 - if (!reached_end)
1.1247 - {
1.1248 - throw new ParseException("Expected end of string; none found");
1.1249 - }
1.1250 - _LexString = builder.ToString();
1.1251 - _LexChar = _LexString.Length > 0 ? _LexString[0] : '\0';
1.1252 - }*/
1.1253 - else if (Contents[0] == '\'')
1.1254 - {
1.1255 - char c = '\0';
1.1256 - if (Contents.Length < 3)
1.1257 - {
1.1258 - throw new ParseException("Expected end of character; none found");
1.1259 - }
1.1260 - else if (Contents[1] == '\\')
1.1261 - {
1.1262 - if (Contents.Length < 4 || !IsEscapedChar(Contents[2]))
1.1263 - {
1.1264 - throw new ParseException("Expected end of character; none found");
1.1265 - }
1.1266 - if (Contents[2] == 'r')
1.1267 - {
1.1268 - c = '\r';
1.1269 - }
1.1270 - else if (Contents[2] == 'n')
1.1271 - {
1.1272 - c = '\n';
1.1273 - }
1.1274 - else if (Contents[2] == 't')
1.1275 - {
1.1276 - c = '\t';
1.1277 - }
1.1278 - else if (Contents[2] == 'v')
1.1279 - {
1.1280 - c = '\v';
1.1281 - }
1.1282 - else if (Contents[2] == '\\')
1.1283 - {
1.1284 - c = '\\';
1.1285 - }
1.1286 - else if (Contents[2] == '"')
1.1287 - {
1.1288 - c = '"';
1.1289 - }
1.1290 - stripchars = 4;
1.1291 - }
1.1292 - else
1.1293 - {
1.1294 - c = Contents[1];
1.1295 - stripchars = 3;
1.1296 - }
1.1297 - _LexString = c.ToString();
1.1298 - _LexChar = c;
1.1299 - _LastToken = LexToken.TOKEN_QUOTCHAR;
1.1300 - }
1.1301 - else if (CheckString(0, "/**"))
1.1302 - {
1.1303 - int endpoint = 0;
1.1304 - for (int i=3; i<Contents.Length-1; i++)
1.1305 - {
1.1306 - if (Contents[i] == '\n')
1.1307 - {
1.1308 - LineNo++;
1.1309 - }
1.1310 - else if (Contents[i] == '*' && Contents[i+1] == '/')
1.1311 - {
1.1312 - endpoint = i+1;
1.1313 - break;
1.1314 - }
1.1315 - }
1.1316 - if (endpoint == 0)
1.1317 - {
1.1318 - throw new ParseException("Expected end of multi-line comment; found end of file");
1.1319 - }
1.1320 - /* We have to factor one more character in because we're zero based */
1.1321 - endpoint++;
1.1322 - _LastToken = LexToken.TOKEN_DOCBLOCK;
1.1323 - _LexString = Contents.ToString().Substring(0, endpoint);
1.1324 - _LexChar = _LexString[0];
1.1325 - stripchars = endpoint;
1.1326 - }
1.1327 - else
1.1328 - {
1.1329 - _LastToken = LexToken.TOKEN_CHARACTER;
1.1330 - _LexString = Contents[0].ToString();
1.1331 - _LexChar = Contents[0];
1.1332 - stripchars = 1;
1.1333 - }
1.1334 - }
1.1335 -
1.1336 - /* Strip N chars */
1.1337 - if (stripchars > 0)
1.1338 - {
1.1339 - Contents.Remove(0, stripchars);
1.1340 - }
1.1341 -
1.1342 - return _LastToken;
1.1343 - }
1.1344 - }
1.1345 -}
