1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/tools/incparser/IncParser.cs Sun Jul 06 04:49:55 2008 +0000
1.3 @@ -0,0 +1,1342 @@
1.4 +using System;
1.5 +using System.Collections.Generic;
1.6 +using System.IO;
1.7 +using System.Text;
1.8 +
1.9 +namespace incparser
1.10 +{
1.11 + enum LexToken
1.12 + {
1.13 + TOKEN_NONE,
1.14 + TOKEN_INCLUDE,
1.15 + TOKEN_DEFINE,
1.16 + TOKEN_CONST,
1.17 + TOKEN_STRUCT,
1.18 + TOKEN_FUNCTAG,
1.19 + TOKEN_FUNCENUM,
1.20 + TOKEN_ENUM,
1.21 + TOKEN_NATIVE,
1.22 + TOKEN_STOCK,
1.23 + TOKEN_PUBLIC,
1.24 + TOKEN_FORWARD,
1.25 + TOKEN_QUOTCHAR,
1.26 + TOKEN_OPERATOR,
1.27 + TOKEN_CHARACTER,
1.28 + /*TOKEN_STRING,*/
1.29 + TOKEN_LABEL,
1.30 + TOKEN_SYMBOL,
1.31 + TOKEN_DOCBLOCK,
1.32 + TOKEN_EOF,
1.33 + }
1.34 +
1.35 + class ParseException : System.Exception
1.36 + {
1.37 + public ParseException(string message) : base(message)
1.38 + {
1.39 + }
1.40 + };
1.41 +
1.42 + class KeywordToken
1.43 + {
1.44 + public KeywordToken(string k, LexToken t)
1.45 + {
1.46 + keyword = k;
1.47 + token = t;
1.48 + }
1.49 + public string keyword;
1.50 + public LexToken token;
1.51 + };
1.52 +
1.53 + class Tokenizer
1.54 + {
1.55 + private static bool s_initialized = false;
1.56 + private static int s_position = 0;
1.57 + private static KeywordToken [] s_tokens = null;
1.58 + public static KeywordToken[] Tokens = null;
1.59 +
1.60 + public static void Initialize()
1.61 + {
1.62 + if (s_initialized)
1.63 + {
1.64 + return;
1.65 + }
1.66 + s_tokens = new KeywordToken[12];
1.67 + s_initialized = true;
1.68 + s_position = 0;
1.69 +
1.70 + AddToken("#include", LexToken.TOKEN_INCLUDE);
1.71 + AddToken("#define", LexToken.TOKEN_DEFINE);
1.72 + AddToken("native", LexToken.TOKEN_NATIVE);
1.73 + AddToken("functag", LexToken.TOKEN_FUNCTAG);
1.74 + AddToken("funcenum", LexToken.TOKEN_FUNCENUM);
1.75 + AddToken("enum", LexToken.TOKEN_ENUM);
1.76 + AddToken("stock", LexToken.TOKEN_STOCK);
1.77 + AddToken("public", LexToken.TOKEN_PUBLIC);
1.78 + AddToken("forward", LexToken.TOKEN_FORWARD);
1.79 + AddToken("const", LexToken.TOKEN_CONST);
1.80 + AddToken("struct", LexToken.TOKEN_STRUCT);
1.81 + AddToken("operator", LexToken.TOKEN_OPERATOR);
1.82 +
1.83 + Tokens = s_tokens;
1.84 + }
1.85 +
1.86 + private static void AddToken(string lex, LexToken tok)
1.87 + {
1.88 + s_tokens[s_position] = new KeywordToken(lex, tok);
1.89 + s_position++;
1.90 + }
1.91 + };
1.92 +
1.93 + class IncParser
1.94 + {
1.95 + private string FileName;
1.96 + private StringBuilder Contents;
1.97 + private bool LexPushed;
1.98 + private LexToken _LastToken;
1.99 + private string _LexString;
1.100 + private char _LexChar;
1.101 + private uint LineNo;
1.102 +
1.103 + public string LexString
1.104 + {
1.105 + get
1.106 + {
1.107 + return _LexString;
1.108 + }
1.109 + }
1.110 +
1.111 + public char LexChar
1.112 + {
1.113 + get
1.114 + {
1.115 + return _LexChar;
1.116 + }
1.117 + }
1.118 +
1.119 + public uint GetLineNumber()
1.120 + {
1.121 + return LineNo;
1.122 + }
1.123 +
1.124 + public IncParser(string file)
1.125 + {
1.126 + FileName = file;
1.127 +
1.128 + /* Clear out lexer stuff */
1.129 + LexPushed = false;
1.130 + _LastToken = LexToken.TOKEN_NONE;
1.131 + _LexString = null;
1.132 + _LexChar = '\0';
1.133 + LineNo = 1;
1.134 +
1.135 + /* Initialize; this can throw an exception */
1.136 + Initialize();
1.137 + }
1.138 +
1.139 + public void Parse(ParseWriter w)
1.140 + {
1.141 + w.BeginSection(System.IO.Path.GetFileName(FileName));
1.142 + LexToken tok = LexToken.TOKEN_NONE;
1.143 + while ((tok = lex()) != LexToken.TOKEN_EOF)
1.144 + {
1.145 + switch (tok)
1.146 + {
1.147 + case LexToken.TOKEN_DOCBLOCK:
1.148 + {
1.149 + w.WritePair("doc", LexString);
1.150 + break;
1.151 + }
1.152 + case LexToken.TOKEN_DEFINE:
1.153 + {
1.154 + PARSE_Define(w);
1.155 + break;
1.156 + }
1.157 + case LexToken.TOKEN_ENUM:
1.158 + {
1.159 + PARSE_Enum(w);
1.160 + break;
1.161 + }
1.162 + case LexToken.TOKEN_FORWARD:
1.163 + {
1.164 + PARSE_Function(tok, w);
1.165 + break;
1.166 + }
1.167 + case LexToken.TOKEN_NATIVE:
1.168 + {
1.169 + PARSE_Function(tok, w);
1.170 + break;
1.171 + }
1.172 + case LexToken.TOKEN_STOCK:
1.173 + {
1.174 + PARSE_Function(tok, w);
1.175 + break;
1.176 + }
1.177 + case LexToken.TOKEN_FUNCTAG:
1.178 + {
1.179 + PARSE_FuncTag(w);
1.180 + break;
1.181 + }
1.182 + case LexToken.TOKEN_FUNCENUM:
1.183 + {
1.184 + PARSE_FuncEnum(w);
1.185 + break;
1.186 + }
1.187 + case LexToken.TOKEN_INCLUDE:
1.188 + {
1.189 + PARSE_Include(w);
1.190 + break;
1.191 + }
1.192 + case LexToken.TOKEN_STRUCT:
1.193 + {
1.194 + PARSE_Struct(w);
1.195 + break;
1.196 + }
1.197 + case LexToken.TOKEN_PUBLIC:
1.198 + {
1.199 + PARSE_Public(w);
1.200 + break;
1.201 + }
1.202 + default:
1.203 + {
1.204 + if (_LexString == "static" || _LexString == "new")
1.205 + {
1.206 + PARSE_Public(w);
1.207 + break;
1.208 + }
1.209 + throw new ParseException("Unrecognized token: " + tok + " " + _LexString);
1.210 + }
1.211 + }
1.212 + }
1.213 + w.EndSection();
1.214 + }
1.215 +
1.216 + private void PARSE_Public(ParseWriter w)
1.217 + {
1.218 + MatchToken(LexToken.TOKEN_CONST);
1.219 +
1.220 + /* Eat up an optional tag */
1.221 + MatchToken(LexToken.TOKEN_LABEL);
1.222 +
1.223 + /* Eat up a name */
1.224 + NeedToken(LexToken.TOKEN_SYMBOL);
1.225 +
1.226 + while (MatchChar('['))
1.227 + {
1.228 + DiscardUntilChar(']');
1.229 + NeedChar(']');
1.230 + }
1.231 +
1.232 + if (MatchChar('='))
1.233 + {
1.234 + ignore_block();
1.235 + }
1.236 + else if (MatchChar('(') && MatchChar(')'))
1.237 + {
1.238 + ignore_block();
1.239 + return;
1.240 + }
1.241 +
1.242 + NeedChar(';');
1.243 + }
1.244 +
1.245 + private void PARSE_Struct(ParseWriter w)
1.246 + {
1.247 + /* For now, we completely ignore these (they're not written to the output) */
1.248 + NeedToken(LexToken.TOKEN_SYMBOL);
1.249 + w.structList.Add(LexString);
1.250 + NeedChar('{');
1.251 +
1.252 + bool need_closebrace = true;
1.253 + bool more_entries = true;
1.254 + do
1.255 + {
1.256 + if (MatchChar('}'))
1.257 + {
1.258 + need_closebrace = false;
1.259 + break;
1.260 + }
1.261 + MatchToken(LexToken.TOKEN_CONST);
1.262 + MatchToken(LexToken.TOKEN_LABEL);
1.263 + NeedToken(LexToken.TOKEN_SYMBOL);
1.264 + while (MatchChar('['))
1.265 + {
1.266 + DiscardUntilChar(']');
1.267 + NeedChar(']');
1.268 + }
1.269 + more_entries = MatchChar(',');
1.270 + MatchToken(LexToken.TOKEN_DOCBLOCK);
1.271 + if (!more_entries)
1.272 + {
1.273 + break;
1.274 + }
1.275 + } while (true);
1.276 + if (need_closebrace)
1.277 + {
1.278 + NeedChar('}');
1.279 + }
1.280 + NeedChar(';');
1.281 + }
1.282 +
1.283 + private void PARSE_Include(ParseWriter w)
1.284 + {
1.285 + /* read until the end of the line */
1.286 + int index = Contents.ToString().IndexOf('\n');
1.287 + string value;
1.288 + if (index == -1)
1.289 + {
1.290 + value = Contents.ToString();
1.291 + Contents.Remove(0, value.Length);
1.292 + }
1.293 + else if (index == 0)
1.294 + {
1.295 + value = "";
1.296 + }
1.297 + else
1.298 + {
1.299 + value = Contents.ToString().Substring(0, index);
1.300 + Contents.Remove(0, index);
1.301 + }
1.302 +
1.303 + /* Strip whitespace */
1.304 + value = value.Trim('\r', '\t', ' ');
1.305 +
1.306 + /* Write */
1.307 + w.BeginSection("include");
1.308 + w.WritePair("name", value);
1.309 + w.EndSection();
1.310 + }
1.311 +
1.312 + private void PARSE_FuncEnum(ParseWriter w)
1.313 + {
1.314 + w.BeginSection("funcenum");
1.315 +
1.316 + /* Get the functag name */
1.317 + NeedToken(LexToken.TOKEN_SYMBOL);
1.318 + w.WritePair("name", LexString);
1.319 + w.funcenumList.Add(LexString);
1.320 +
1.321 + NeedChar('{');
1.322 +
1.323 + bool need_closebrace = true;
1.324 + do
1.325 + {
1.326 + /* Shortcut out? */
1.327 + if (MatchChar('}'))
1.328 + {
1.329 + need_closebrace = false;
1.330 + break;
1.331 + }
1.332 +
1.333 + /* Start function section */
1.334 + w.BeginSection("function");
1.335 +
1.336 + if (MatchToken(LexToken.TOKEN_DOCBLOCK))
1.337 + {
1.338 + w.WritePair("doc", LexString);
1.339 + }
1.340 +
1.341 + /* Get the return tag */
1.342 + if (MatchToken(LexToken.TOKEN_LABEL))
1.343 + {
1.344 + w.WritePair("return", LexString);
1.345 + }
1.346 +
1.347 + /* Get function type */
1.348 + if (MatchToken(LexToken.TOKEN_PUBLIC))
1.349 + {
1.350 + w.WritePair("type", "public");
1.351 + }
1.352 + else if (MatchToken(LexToken.TOKEN_STOCK))
1.353 + {
1.354 + w.WritePair("type", "stock");
1.355 + }
1.356 +
1.357 + /* Parse the parameters */
1.358 + ParseParameters(w);
1.359 +
1.360 + /* End the section */
1.361 + w.EndSection();
1.362 +
1.363 + if (!MatchChar(','))
1.364 + {
1.365 + break;
1.366 + }
1.367 + } while (true);
1.368 +
1.369 + if (need_closebrace)
1.370 + {
1.371 + NeedChar('}');
1.372 + }
1.373 + NeedChar(';');
1.374 +
1.375 + w.EndSection();
1.376 + }
1.377 +
1.378 + private void PARSE_FuncTag(ParseWriter w)
1.379 + {
1.380 + w.BeginSection("functag");
1.381 +
1.382 + /* Get the functag name */
1.383 + NeedToken(LexToken.TOKEN_SYMBOL);
1.384 + w.WritePair("name", LexString);
1.385 + w.functagList.Add(LexString);
1.386 +
1.387 + if (MatchToken(LexToken.TOKEN_LABEL))
1.388 + {
1.389 + w.WritePair("return", LexString);
1.390 + }
1.391 +
1.392 + if (MatchToken(LexToken.TOKEN_PUBLIC))
1.393 + {
1.394 + w.WritePair("type", "public");
1.395 + }
1.396 + else if (MatchToken(LexToken.TOKEN_STOCK))
1.397 + {
1.398 + w.WritePair("type", "stock");
1.399 + }
1.400 +
1.401 + ParseParameters(w);
1.402 + NeedChar(';');
1.403 +
1.404 + w.EndSection();
1.405 + }
1.406 +
1.407 + private void PARSE_Function(LexToken tok, ParseWriter w)
1.408 + {
1.409 + string tag="", name;
1.410 +
1.411 + /* Get the return value */
1.412 + if (MatchToken(LexToken.TOKEN_LABEL))
1.413 + {
1.414 + tag = LexString;
1.415 + }
1.416 +
1.417 + if (MatchToken(LexToken.TOKEN_OPERATOR))
1.418 + {
1.419 + /* Check if we're some sort of invalid function */
1.420 + DiscardUntilCharOrComment('(', false);
1.421 + ParseParameters(null);
1.422 + if (tok == LexToken.TOKEN_STOCK)
1.423 + {
1.424 + ignore_block();
1.425 + }
1.426 + else if (tok == LexToken.TOKEN_NATIVE || tok == LexToken.TOKEN_FORWARD)
1.427 + {
1.428 + if (MatchChar('='))
1.429 + {
1.430 + DiscardUntilCharOrComment(';', false);
1.431 + }
1.432 + NeedChar(';');
1.433 + }
1.434 + return;
1.435 + }
1.436 +
1.437 + /* Get the name */
1.438 + NeedToken(LexToken.TOKEN_SYMBOL);
1.439 + name = LexString;
1.440 +
1.441 + PARSE_Function(tok, w, tag, name);
1.442 + }
1.443 +
1.444 + private void PARSE_Function(LexToken tok, ParseWriter w, string tag, string name)
1.445 + {
1.446 + if (tok == LexToken.TOKEN_FORWARD)
1.447 + {
1.448 + w.BeginSection("forward");
1.449 + w.forwardList.Add(name);
1.450 + }
1.451 + else if (tok == LexToken.TOKEN_NATIVE)
1.452 + {
1.453 + w.BeginSection("native");
1.454 + w.nativeList.Add(name);
1.455 + }
1.456 + else if (tok == LexToken.TOKEN_STOCK)
1.457 + {
1.458 + w.BeginSection("stock");
1.459 + w.stockList.Add(name);
1.460 + }
1.461 +
1.462 + w.WritePair("name", name);
1.463 +
1.464 + if (tag.Length > 0)
1.465 + {
1.466 + w.WritePair("return", tag);
1.467 + }
1.468 +
1.469 + ParseParameters(w);
1.470 +
1.471 + if (tok == LexToken.TOKEN_STOCK)
1.472 + {
1.473 + ignore_block();
1.474 + }
1.475 + else
1.476 + {
1.477 + /* Make sure there's a semicolon */
1.478 + if (MatchChar('='))
1.479 + {
1.480 + DiscardUntilCharOrComment(';', false);
1.481 + }
1.482 + NeedChar(';');
1.483 + }
1.484 +
1.485 + w.EndSection();
1.486 + }
1.487 +
1.488 + private void ParseParameters(ParseWriter w)
1.489 + {
1.490 + NeedChar('(');
1.491 + string extra;
1.492 + do
1.493 + {
1.494 + if (MatchChar(')'))
1.495 + {
1.496 + return;
1.497 + }
1.498 +
1.499 + /* Start the parameter */
1.500 + if (w != null)
1.501 + {
1.502 + w.BeginSection("parameter");
1.503 + }
1.504 +
1.505 + /* Check for a 'const' token */
1.506 + if (MatchToken(LexToken.TOKEN_CONST) && w != null)
1.507 + {
1.508 + w.WritePair("const", "true");
1.509 + }
1.510 +
1.511 + /* Check if it's by reference */
1.512 + if (MatchChar('&') && w != null)
1.513 + {
1.514 + w.WritePair("byref", "true");
1.515 + }
1.516 +
1.517 + /* Check for a tag */
1.518 + if (MatchToken(LexToken.TOKEN_LABEL) && w != null)
1.519 + {
1.520 + w.WritePair("tag", LexString);
1.521 + }
1.522 +
1.523 + /* Get the symbol and write it */
1.524 + if (MatchString(0, "...") && w != null)
1.525 + {
1.526 + w.WritePair("name", "...");
1.527 + }
1.528 + else
1.529 + {
1.530 + NeedToken(LexToken.TOKEN_SYMBOL);
1.531 + if (w != null)
1.532 + {
1.533 + w.WritePair("name", LexString);
1.534 + }
1.535 + }
1.536 +
1.537 + /* Check if we have a default value yet */
1.538 + bool default_value = false;
1.539 + if (MatchChar('='))
1.540 + {
1.541 + default_value = true;
1.542 + }
1.543 + else if (MatchChar('['))
1.544 + {
1.545 + extra = "";
1.546 + string temp;
1.547 + do
1.548 + {
1.549 + extra += "[";
1.550 + temp = ConsumeUntilChar(']');
1.551 + if (temp != null)
1.552 + {
1.553 + extra += temp;
1.554 + }
1.555 + extra += "]";
1.556 + NeedChar(']');
1.557 + } while (MatchChar('['));
1.558 + w.WritePair("decoration", extra);
1.559 + }
1.560 +
1.561 + /* If we have a default value, get it */
1.562 + if (default_value || MatchChar('='))
1.563 + {
1.564 + if ((extra = ConsumeParamDefValue()) == null)
1.565 + {
1.566 + throw new ParseException("Expected a default value; found none");
1.567 + }
1.568 + if (w != null)
1.569 + {
1.570 + w.WritePair("defval", extra);
1.571 + }
1.572 + }
1.573 +
1.574 + /* End the parameter */
1.575 + if (w != null)
1.576 + {
1.577 + w.EndSection();
1.578 + }
1.579 +
1.580 + if (!MatchChar(','))
1.581 + {
1.582 + break;
1.583 + }
1.584 + } while (true);
1.585 +
1.586 + NeedChar(')');
1.587 + }
1.588 +
1.589 + private void PARSE_Enum(ParseWriter w)
1.590 + {
1.591 + string name = "";
1.592 + if (MatchToken(LexToken.TOKEN_SYMBOL))
1.593 + {
1.594 + name = LexString;
1.595 + }
1.596 +
1.597 + w.BeginSection("enum");
1.598 + w.WritePair("name", name);
1.599 + w.BeginSection("properties");
1.600 +
1.601 + w.enumList.Add(name);
1.602 +
1.603 + NeedChar('{');
1.604 + bool more_entries = true;
1.605 + bool need_closebrace = true;
1.606 +
1.607 + if (MatchToken(LexToken.TOKEN_DOCBLOCK))
1.608 + {
1.609 + w.WritePair("doc", LexString);
1.610 + }
1.611 +
1.612 + do
1.613 + {
1.614 + if (MatchChar('}'))
1.615 + {
1.616 + need_closebrace = false;
1.617 + break;
1.618 + }
1.619 + NeedToken(LexToken.TOKEN_SYMBOL);
1.620 + name = LexString;
1.621 +
1.622 + if (MatchChar('='))
1.623 + {
1.624 + DiscardUntilCharOrComment(",\n}", true);
1.625 + }
1.626 + more_entries = MatchChar(',');
1.627 + w.WritePair("name", name);
1.628 + w.enumTypeList.Add(name);
1.629 + if (MatchToken(LexToken.TOKEN_DOCBLOCK))
1.630 + {
1.631 + w.WritePair("doc", LexString);
1.632 + }
1.633 + } while (more_entries);
1.634 + if (need_closebrace)
1.635 + {
1.636 + NeedChar('}');
1.637 + }
1.638 + NeedChar(';');
1.639 + w.EndSection();
1.640 + w.EndSection();
1.641 + }
1.642 +
1.643 + private void PARSE_Define(ParseWriter w)
1.644 + {
1.645 + /* first, get the symbol name */
1.646 + NeedToken(LexToken.TOKEN_SYMBOL);
1.647 + string name = LexString;
1.648 + string value;
1.649 +
1.650 + /* read until we hit the end of a line OR a comment */
1.651 + int endpoint = -1;
1.652 + for (int i = 0; i < Contents.Length; i++)
1.653 + {
1.654 + if (Contents[i] == '\n')
1.655 + {
1.656 + endpoint = i;
1.657 + break;
1.658 + }
1.659 + else if (CheckString(i, "/**"))
1.660 + {
1.661 + endpoint = i;
1.662 + break;
1.663 + }
1.664 + }
1.665 +
1.666 + if (endpoint < 1)
1.667 + {
1.668 + value = "";
1.669 + }
1.670 + else
1.671 + {
1.672 + value = Contents.ToString().Substring(0, endpoint);
1.673 + Contents.Remove(0, endpoint);
1.674 + }
1.675 +
1.676 + /* Write */
1.677 + w.defineList.Add(name);
1.678 +
1.679 + w.BeginSection("define");
1.680 + w.WritePair("name", name);
1.681 + w.WritePair("linetext", value);
1.682 + w.EndSection();
1.683 + }
1.684 +
1.685 + /* Reads up to a character and discards all text before it.
1.686 + */
1.687 + private void DiscardUntilChar(char c)
1.688 + {
1.689 + int eol = Contents.ToString().IndexOf(c);
1.690 +
1.691 + if (eol < 1)
1.692 + {
1.693 + return;
1.694 + }
1.695 +
1.696 + Contents.Remove(0, eol);
1.697 + }
1.698 +
1.699 + /* Same as DiscardUntilChar, except the discarded
1.700 + * text is returned first.
1.701 + */
1.702 + private string ConsumeUntilChar(char c)
1.703 + {
1.704 + int eol = Contents.ToString().IndexOf(c);
1.705 +
1.706 + if (eol < 1)
1.707 + {
1.708 + return null;
1.709 + }
1.710 +
1.711 + string rest = Contents.ToString().Substring(0, eol);
1.712 + Contents.Remove(0, eol);
1.713 + return rest;
1.714 + }
1.715 +
1.716 + /**
1.717 + * Returns a parameter's default value.
1.718 + */
1.719 + private string ConsumeParamDefValue()
1.720 + {
1.721 + int eol = FindEndOfExprLine(0);
1.722 + if (eol < 1)
1.723 + {
1.724 + return null;
1.725 + }
1.726 +
1.727 + string rest = Contents.ToString().Substring(0, eol);
1.728 + Contents.Remove(0, eol);
1.729 + return rest;
1.730 + }
1.731 +
1.732 + /**
1.733 + * Discards all data up to a certain character, ignoring
1.734 + * the contents, unless specified to stop at a doc block comment.
1.735 + */
1.736 + private void DiscardUntilCharOrComment(char c, bool stop_at_comment)
1.737 + {
1.738 + for (int i = 0; i < Contents.Length; i++)
1.739 + {
1.740 + if (Contents[i] == '\n')
1.741 + {
1.742 + LineNo++;
1.743 + }
1.744 + else if (Contents[i] == c)
1.745 + {
1.746 + if (i > 0)
1.747 + {
1.748 + Contents.Remove(0, i);
1.749 + }
1.750 + return;
1.751 + }
1.752 + else if (stop_at_comment && CheckString(i, "/**"))
1.753 + {
1.754 + if (i > 0)
1.755 + {
1.756 + Contents.Remove(0, i);
1.757 + }
1.758 + return;
1.759 + }
1.760 + }
1.761 + }
1.762 +
1.763 + /**
1.764 + * Discards all data up to a certain character, ignoring
1.765 + * the contents, unless specified to stop at a doc block comment.
1.766 + */
1.767 + private void DiscardUntilCharOrComment(string s, bool stop_at_comment)
1.768 + {
1.769 + for (int i = 0; i < Contents.Length; i++)
1.770 + {
1.771 + if (Contents[i] == '\n')
1.772 + {
1.773 + LineNo++;
1.774 + }
1.775 + else if (s.Contains(Contents[i].ToString()))
1.776 + {
1.777 + if (i > 0)
1.778 + {
1.779 + Contents.Remove(0, i);
1.780 + }
1.781 + return;
1.782 + }
1.783 + else if (stop_at_comment && CheckString(i, "/**"))
1.784 + {
1.785 + if (i > 0)
1.786 + {
1.787 + Contents.Remove(0, i);
1.788 + }
1.789 + return;
1.790 + }
1.791 + }
1.792 + }
1.793 +
1.794 + /**
1.795 + * Finds the end of an expression
1.796 + */
1.797 + private int FindEndOfExprLine(int start)
1.798 + {
1.799 + for (int i = start; i < Contents.Length; i++)
1.800 + {
1.801 + if (Contents[i] == '{')
1.802 + {
1.803 + if ((i = FindEndOfConsumptionLine('}', start+1)) == -1)
1.804 + {
1.805 + throw new ParseException("Expected token: } (found none)");
1.806 + }
1.807 + }
1.808 + else if (Contents[i] == ')' || Contents[i] == ',')
1.809 + {
1.810 + return i;
1.811 + }
1.812 + }
1.813 +
1.814 + return -1;
1.815 + }
1.816 +
1.817 + private int FindEndOfConsumptionLine(char c, int start)
1.818 + {
1.819 + for (int i = start; i < Contents.Length; i++)
1.820 + {
1.821 + if (Contents[i] == '{')
1.822 + {
1.823 + if ((i = FindEndOfConsumptionLine('}', i + 1)) == -1)
1.824 + {
1.825 + throw new ParseException("Expected token: } (found none)");
1.826 + }
1.827 + }
1.828 + else if (Contents[i] == c)
1.829 + {
1.830 + return i;
1.831 + }
1.832 + }
1.833 +
1.834 + return -1;
1.835 + }
1.836 +
1.837 + private void Initialize()
1.838 + {
1.839 + /* Open file, read it, and close it */
1.840 + StreamReader sr = File.OpenText(FileName);
1.841 + Contents = new StringBuilder(sr.ReadToEnd());
1.842 + sr.Close();
1.843 +
1.844 + /* Strip comments (may throw an exception) */
1.845 + preprocess();
1.846 +
1.847 + /* Lastly, initialize the tokenizer */
1.848 + Tokenizer.Initialize();
1.849 + }
1.850 +
1.851 + /* Consumes a token iff it matches */
1.852 + private bool MatchToken(LexToken token)
1.853 + {
1.854 + LexToken tok = lex();
1.855 + if (tok != token)
1.856 + {
1.857 + lexpush();
1.858 + return false;
1.859 + }
1.860 + return true;
1.861 + }
1.862 +
1.863 + /* Consumes a token and aborts the parser if there is no match */
1.864 + private void NeedToken(LexToken token)
1.865 + {
1.866 + if (!MatchToken(token))
1.867 + {
1.868 + throw new ParseException("Expected token: " + token);
1.869 + }
1.870 + }
1.871 +
1.872 + /* Consumes a character iff it matches */
1.873 + private bool MatchChar(char c)
1.874 + {
1.875 + if (lex() != LexToken.TOKEN_CHARACTER)
1.876 + {
1.877 + lexpush();
1.878 + return false;
1.879 + }
1.880 + if (_LexChar != c)
1.881 + {
1.882 + lexpush();
1.883 + return false;
1.884 + }
1.885 + return true;
1.886 + }
1.887 +
1.888 + /* Consumes a character or aborts the parser if there is no match */
1.889 + private void NeedChar(char c)
1.890 + {
1.891 + if (!MatchChar(c))
1.892 + {
1.893 + throw new ParseException("Expected character: " + c);
1.894 + }
1.895 + }
1.896 +
1.897 + /* Saves the lexer state so we don't reparse unnecessarily */
1.898 + private void lexpush()
1.899 + {
1.900 + if (LexPushed)
1.901 + {
1.902 + throw new ParseException("Lexer cannot be pushed twice");
1.903 + }
1.904 + LexPushed = true;
1.905 + }
1.906 +
1.907 + /* Attempts to match a string against the next stream tokens.
1.908 + * On success, the token stream is advanced.
1.909 + */
1.910 + private bool MatchString(int startPos, string str)
1.911 + {
1.912 + if (str.Length > Contents.Length - startPos)
1.913 + {
1.914 + return false;
1.915 + }
1.916 + for (int i = 0; i < str.Length; i++)
1.917 + {
1.918 + if (str[i] != Contents[startPos + i])
1.919 + {
1.920 + return false;
1.921 + }
1.922 + }
1.923 + Contents.Remove(startPos, str.Length);
1.924 + return true;
1.925 + }
1.926 +
1.927 + /* Attempts to match a string against the next stream tokens */
1.928 + private bool CheckString(int startPos, string str)
1.929 + {
1.930 + if (str.Length > Contents.Length - startPos)
1.931 + {
1.932 + return false;
1.933 + }
1.934 + for (int i = 0; i < str.Length; i++)
1.935 + {
1.936 + if (str[i] != Contents[startPos + i])
1.937 + {
1.938 + return false;
1.939 + }
1.940 + }
1.941 + return true;
1.942 + }
1.943 +
1.944 + /* Ignores a block starting and ending with { and } respectively */
1.945 + private void ignore_block()
1.946 + {
1.947 + /* Hrm, this is tricky. First, expect an open brace. */
1.948 + NeedChar('{');
1.949 + /* Consume the entire function by keeping track of braces */
1.950 + int numBraces = 1;
1.951 + bool in_string = false;
1.952 + bool found_end = false;
1.953 + for (int i = 0; i < Contents.Length; i++)
1.954 + {
1.955 + if (Contents[i] == '\n')
1.956 + {
1.957 + LineNo++;
1.958 + }
1.959 + if (!in_string)
1.960 + {
1.961 + if (Contents[i] == '"')
1.962 + {
1.963 + in_string = true;
1.964 + }
1.965 + else if (Contents[i] == '{')
1.966 + {
1.967 + numBraces++;
1.968 + }
1.969 + else if (Contents[i] == '}')
1.970 + {
1.971 + if (numBraces == 0)
1.972 + {
1.973 + throw new ParseException("Unable to find a matching open brace");
1.974 + }
1.975 + if (--numBraces == 0)
1.976 + {
1.977 + Contents.Remove(0, i + 1);
1.978 + found_end = true;
1.979 + break;
1.980 + }
1.981 + }
1.982 + }
1.983 + else
1.984 + {
1.985 + /* It is possible to be in a string and at the first character */
1.986 + if (Contents[i] == '"' && Contents[i - 1] != '\\')
1.987 + {
1.988 + in_string = false;
1.989 + }
1.990 + }
1.991 + }
1.992 + if (!found_end)
1.993 + {
1.994 + throw new ParseException("Unable to find a matching close brace");
1.995 + }
1.996 + }
1.997 +
1.998 + /* Strips all non-doc C and C++ style comments. as well as unused pragmas */
1.999 + private void preprocess()
1.1000 + {
1.1001 + if (Contents.Length == 1)
1.1002 + {
1.1003 + return;
1.1004 + }
1.1005 +
1.1006 + bool in_comment = false;
1.1007 + bool is_ml = false;
1.1008 + bool is_doc = false;
1.1009 + bool is_newline = true;
1.1010 +
1.1011 + for (int i=0; i<Contents.Length; i++)
1.1012 + {
1.1013 + if (Contents[i] == '\n')
1.1014 + {
1.1015 + is_newline = true;
1.1016 + }
1.1017 + if (!in_comment)
1.1018 + {
1.1019 + if (Contents[i] == '/')
1.1020 + {
1.1021 + if (Contents[i+1] == '/')
1.1022 + {
1.1023 + in_comment = true;
1.1024 + is_ml = false;
1.1025 + is_doc = false;
1.1026 + is_newline = false;
1.1027 + } else if (Contents[i+1] == '*') {
1.1028 + /* Detect whether this is a doc comment or not */
1.1029 + if (Contents.Length - i >= 3
1.1030 + && Contents[i+2] == '*')
1.1031 + {
1.1032 + is_doc = true;
1.1033 + } else {
1.1034 + is_doc = false;
1.1035 + }
1.1036 + in_comment = true;
1.1037 + is_ml = true;
1.1038 + }
1.1039 + if (in_comment && !is_doc)
1.1040 + {
1.1041 + Contents[i] = ' ';
1.1042 + Contents[i+1] = ' '; /* Just for safety */
1.1043 + }
1.1044 + }
1.1045 + else if (is_newline)
1.1046 + {
1.1047 + if (Contents[i] == '#')
1.1048 + {
1.1049 + if (!CheckString(i, "#define")
1.1050 + && !CheckString(i, "#include"))
1.1051 + {
1.1052 + /* Ignore! */
1.1053 + Contents[i] = ' ';
1.1054 + in_comment = true;
1.1055 + is_doc = false;
1.1056 + is_ml = false;
1.1057 + }
1.1058 + is_newline = false;
1.1059 + }
1.1060 + else if (!Char.IsWhiteSpace(Contents[i]))
1.1061 + {
1.1062 + is_newline = false;
1.1063 + }
1.1064 + }
1.1065 + } else {
1.1066 + if (is_ml)
1.1067 + {
1.1068 + if (Contents[i] == '*' && Contents[i+1] == '/')
1.1069 + {
1.1070 + if (!is_doc)
1.1071 + {
1.1072 + Contents[i] = ' ';
1.1073 + Contents[i+1] = ' ';
1.1074 + }
1.1075 + in_comment = false;
1.1076 + i++; /* Don't look at the next character */
1.1077 + }
1.1078 + else if (!is_doc && Contents[i] != '\n')
1.1079 + {
1.1080 + /* Erase comment */
1.1081 + Contents[i] = ' ';
1.1082 + }
1.1083 + } else {
1.1084 + if (is_newline)
1.1085 + {
1.1086 + in_comment = false;
1.1087 + }
1.1088 + else
1.1089 + {
1.1090 + /* Erase comment */
1.1091 + Contents[i] = ' ';
1.1092 + }
1.1093 + }
1.1094 + }
1.1095 + }
1.1096 +
1.1097 + if (in_comment)
1.1098 + {
1.1099 + throw new ParseException("Multi-line comment has no terminator");
1.1100 + }
1.1101 + }
1.1102 +
1.1103 + /* Strips whitespace */
1.1104 + private void trim_whitespace_pre()
1.1105 + {
1.1106 + int spaces = 0;
1.1107 + while (spaces < Contents.Length
1.1108 + && Contents[spaces] != '\0'
1.1109 + && Char.IsWhiteSpace(Contents[spaces]))
1.1110 + {
1.1111 + if (Contents[spaces] == '\n')
1.1112 + {
1.1113 + LineNo++;
1.1114 + }
1.1115 + spaces++;
1.1116 + }
1.1117 + if (spaces > 0)
1.1118 + {
1.1119 + Contents.Remove(0, spaces);
1.1120 + }
1.1121 + }
1.1122 +
1.1123 + private void preprocess_line()
1.1124 + {
1.1125 + trim_whitespace_pre();
1.1126 + }
1.1127 +
1.1128 + private bool IsSymbolChar(char c, bool first)
1.1129 + {
1.1130 + bool valid = (first ? Char.IsLetter(c) : Char.IsLetterOrDigit(c));
1.1131 + return (valid || (c == '_'));
1.1132 + }
1.1133 +
1.1134 + private bool IsEscapedChar(char c)
1.1135 + {
1.1136 + return (c == 'n'
1.1137 + || c == 'r'
1.1138 + || c == 't'
1.1139 + || c == 'v'
1.1140 + || c == '\\'
1.1141 + || c == '"');
1.1142 + }
1.1143 +
1.1144 + private bool IsNotKeywordChar(char c)
1.1145 + {
1.1146 + return (Char.IsWhiteSpace(c)
1.1147 + || (c == '-' || c == '+')
1.1148 + || (c == '/' || c == '*')
1.1149 + || (c == '(' || c == ')')
1.1150 + || (c == '=' || c == '%')
1.1151 + || (c == '>' || c == '<')
1.1152 + || (c == '!' || c == '~')
1.1153 + );
1.1154 + }
1.1155 +
1.1156 + /* Consumes one lexical token and gathers information about it */
1.1157 + private LexToken lex()
1.1158 + {
1.1159 + if (LexPushed)
1.1160 + {
1.1161 + LexPushed = false;
1.1162 + return _LastToken;
1.1163 + }
1.1164 +
1.1165 + /* Number of chars we will be deleting from the input */
1.1166 + int stripchars = 0;
1.1167 +
1.1168 + /* Clear our state */
1.1169 + _LastToken = LexToken.TOKEN_NONE;
1.1170 +
1.1171 + /* Remove stuff we don't want from the line */
1.1172 + preprocess_line();
1.1173 +
1.1174 + if (Contents.Length < 1)
1.1175 + {
1.1176 + _LastToken = LexToken.TOKEN_EOF;
1.1177 + return _LastToken;
1.1178 + }
1.1179 +
1.1180 + /* Get the token list */
1.1181 + KeywordToken[] tokens = Tokenizer.Tokens;
1.1182 + for (int i = 0; i < tokens.Length; i++)
1.1183 + {
1.1184 + if (!CheckString(0, tokens[i].keyword))
1.1185 + {
1.1186 + continue;
1.1187 + }
1.1188 + int len = tokens[i].keyword.Length;
1.1189 + /* Now check to see what the next token is */
1.1190 + if (Contents.Length == len
1.1191 + || IsNotKeywordChar(Contents[len]))
1.1192 + {
1.1193 + /* We have a token match! */
1.1194 + _LastToken = tokens[i].token;
1.1195 + _LexString = null;
1.1196 + _LexChar = '\0';
1.1197 + stripchars = len;
1.1198 + break;
1.1199 + }
1.1200 + }
1.1201 +
1.1202 + if (_LastToken == LexToken.TOKEN_NONE)
1.1203 + {
1.1204 + /* See if we can try to read a symbol */
1.1205 + if (IsSymbolChar(Contents[0], true))
1.1206 + {
1.1207 + int characters = 1;
1.1208 + while (Contents[characters] != '\0'
1.1209 + && IsSymbolChar(Contents[characters], false))
1.1210 + {
1.1211 + characters++;
1.1212 + }
1.1213 + stripchars = characters;
1.1214 + /* We're done! See what's next.. */
1.1215 + if (Contents[characters] == ':')
1.1216 + {
1.1217 + _LastToken = LexToken.TOKEN_LABEL;
1.1218 + stripchars++;
1.1219 + }
1.1220 + else
1.1221 + {
1.1222 + _LastToken = LexToken.TOKEN_SYMBOL;
1.1223 + }
1.1224 + _LexString = Contents.ToString().Substring(0, characters);
1.1225 + _LexChar = _LexString[0];
1.1226 + }/*
1.1227 + else if (Contents[0] == '"')
1.1228 + {
1.1229 + bool reached_end = false;
1.1230 + StringBuilder builder = new StringBuilder();
1.1231 + for (int i = 1; i < Contents.Length; i++)
1.1232 + {
1.1233 + if (i < Contents.Length - 1
1.1234 + && Contents[i] == '\\'
1.1235 + && IsEscapedChar(Contents[i+1]))
1.1236 + {
1.1237 + i++;
1.1238 + }
1.1239 + else if (Contents[i] == '"')
1.1240 + {
1.1241 + reached_end = true;
1.1242 + break;
1.1243 + }
1.1244 + builder.Append(Contents[i]);
1.1245 + }
1.1246 + if (!reached_end)
1.1247 + {
1.1248 + throw new ParseException("Expected end of string; none found");
1.1249 + }
1.1250 + _LexString = builder.ToString();
1.1251 + _LexChar = _LexString.Length > 0 ? _LexString[0] : '\0';
1.1252 + }*/
1.1253 + else if (Contents[0] == '\'')
1.1254 + {
1.1255 + char c = '\0';
1.1256 + if (Contents.Length < 3)
1.1257 + {
1.1258 + throw new ParseException("Expected end of character; none found");
1.1259 + }
1.1260 + else if (Contents[1] == '\\')
1.1261 + {
1.1262 + if (Contents.Length < 4 || !IsEscapedChar(Contents[2]))
1.1263 + {
1.1264 + throw new ParseException("Expected end of character; none found");
1.1265 + }
1.1266 + if (Contents[2] == 'r')
1.1267 + {
1.1268 + c = '\r';
1.1269 + }
1.1270 + else if (Contents[2] == 'n')
1.1271 + {
1.1272 + c = '\n';
1.1273 + }
1.1274 + else if (Contents[2] == 't')
1.1275 + {
1.1276 + c = '\t';
1.1277 + }
1.1278 + else if (Contents[2] == 'v')
1.1279 + {
1.1280 + c = '\v';
1.1281 + }
1.1282 + else if (Contents[2] == '\\')
1.1283 + {
1.1284 + c = '\\';
1.1285 + }
1.1286 + else if (Contents[2] == '"')
1.1287 + {
1.1288 + c = '"';
1.1289 + }
1.1290 + stripchars = 4;
1.1291 + }
1.1292 + else
1.1293 + {
1.1294 + c = Contents[1];
1.1295 + stripchars = 3;
1.1296 + }
1.1297 + _LexString = c.ToString();
1.1298 + _LexChar = c;
1.1299 + _LastToken = LexToken.TOKEN_QUOTCHAR;
1.1300 + }
1.1301 + else if (CheckString(0, "/**"))
1.1302 + {
1.1303 + int endpoint = 0;
1.1304 + for (int i=3; i<Contents.Length-1; i++)
1.1305 + {
1.1306 + if (Contents[i] == '\n')
1.1307 + {
1.1308 + LineNo++;
1.1309 + }
1.1310 + else if (Contents[i] == '*' && Contents[i+1] == '/')
1.1311 + {
1.1312 + endpoint = i+1;
1.1313 + break;
1.1314 + }
1.1315 + }
1.1316 + if (endpoint == 0)
1.1317 + {
1.1318 + throw new ParseException("Expected end of multi-line comment; found end of file");
1.1319 + }
1.1320 + /* We have to factor one more character in because we're zero based */
1.1321 + endpoint++;
1.1322 + _LastToken = LexToken.TOKEN_DOCBLOCK;
1.1323 + _LexString = Contents.ToString().Substring(0, endpoint);
1.1324 + _LexChar = _LexString[0];
1.1325 + stripchars = endpoint;
1.1326 + }
1.1327 + else
1.1328 + {
1.1329 + _LastToken = LexToken.TOKEN_CHARACTER;
1.1330 + _LexString = Contents[0].ToString();
1.1331 + _LexChar = Contents[0];
1.1332 + stripchars = 1;
1.1333 + }
1.1334 + }
1.1335 +
1.1336 + /* Strip N chars */
1.1337 + if (stripchars > 0)
1.1338 + {
1.1339 + Contents.Remove(0, stripchars);
1.1340 + }
1.1341 +
1.1342 + return _LastToken;
1.1343 + }
1.1344 + }
1.1345 +}
