core/TextParsers.cpp
author David Anderson <dvander@alliedmods.net>
Mon Dec 10 22:20:53 2007 +0000 (2007-12-10)
changeset 1783 e6e2f440376d
parent 1730 62dd251bb1cd
child 1815 b0de032adc8f
permissions -rw-r--r--
- fixed amb1237 part 1 - maplists.cfg missing could cause ReadMapList() to fail
- fixed amb1237 part 2 - nextmap could fail in certain cases with mapchooser present
     1 /**
     2  * vim: set ts=4 :
     3  * =============================================================================
     4  * SourceMod
     5  * Copyright (C) 2004-2007 AlliedModders LLC.  All rights reserved.
     6  * =============================================================================
     7  *
     8  * This program is free software; you can redistribute it and/or modify it under
     9  * the terms of the GNU General Public License, version 3.0, as published by the
    10  * Free Software Foundation.
    11  * 
    12  * This program is distributed in the hope that it will be useful, but WITHOUT
    13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    14  * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
    15  * details.
    16  *
    17  * You should have received a copy of the GNU General Public License along with
    18  * this program.  If not, see <http://www.gnu.org/licenses/>.
    19  *
    20  * As a special exception, AlliedModders LLC gives you permission to link the
    21  * code of this program (as well as its derivative works) to "Half-Life 2," the
    22  * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software
    23  * by the Valve Corporation.  You must obey the GNU General Public License in
    24  * all respects for all other code used.  Additionally, AlliedModders LLC grants
    25  * this exception to all derivative works.  AlliedModders LLC defines further
    26  * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),
    27  * or <http://www.sourcemod.net/license.php>.
    28  *
    29  * Version: $Id$
    30  */
    31 
    32 #include <stdio.h>
    33 #include <ctype.h>
    34 #include <wctype.h>
    35 #include <string.h>
    36 #include <stdlib.h>
    37 #include <assert.h>
    38 #include "TextParsers.h"
    39 #include "ShareSys.h"
    40 
    41 TextParsers g_TextParser;
    42 ITextParsers *textparsers = &g_TextParser;
    43 
    44 static int g_ini_chartable1[255] = {0};
    45 static int g_ws_chartable[255] = {0};
    46 
    47 bool TextParsers::IsWhitespace(const char *stream)
    48 {
    49 	return g_ws_chartable[(unsigned)*stream] == 1;
    50 }
    51 
    52 TextParsers::TextParsers()
    53 {
    54 	g_ini_chartable1[(unsigned)'_'] = 1;
    55 	g_ini_chartable1[(unsigned)'-'] = 1;
    56 	g_ini_chartable1[(unsigned)','] = 1;
    57 	g_ini_chartable1[(unsigned)'+'] = 1;
    58 	g_ini_chartable1[(unsigned)'.'] = 1;
    59 	g_ini_chartable1[(unsigned)'$'] = 1;
    60 	g_ini_chartable1[(unsigned)'?'] = 1;
    61 	g_ini_chartable1[(unsigned)'/'] = 1;
    62 	g_ws_chartable[(unsigned)'\n'] = 1;
    63 	g_ws_chartable[(unsigned)'\v'] = 1;
    64 	g_ws_chartable[(unsigned)'\r'] = 1;
    65 	g_ws_chartable[(unsigned)'\t'] = 1;
    66 	g_ws_chartable[(unsigned)'\f'] = 1;
    67 	g_ws_chartable[(unsigned)' '] = 1;
    68 }
    69 
    70 void TextParsers::OnSourceModAllInitialized()
    71 {
    72 	g_ShareSys.AddInterface(NULL, this);
    73 }
    74 
    75 unsigned int TextParsers::GetUTF8CharBytes(const char *stream)
    76 {
    77 	return _GetUTF8CharBytes(stream);
    78 }
    79 
    80 /**
    81  * Character streams
    82  */
    83 
    84 struct CharStream
    85 {
    86 	const char *curpos;
    87 };
    88 
    89 bool CharStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
    90 {
    91 	CharStream *srdr = (CharStream *)stream;
    92 
    93 	const char *ptr = srdr->curpos;
    94 	for (size_t i=0; i<maxlength; i++)
    95 	{
    96 		if (*ptr == '\0')
    97 		{
    98 			break;
    99 		}
   100 		*buffer++ = *ptr++;
   101 	}
   102 
   103 	*read = ptr - srdr->curpos;
   104 
   105 	srdr->curpos = ptr;
   106 
   107 	return true;
   108 }
   109 
   110 SMCError TextParsers::ParseString_SMC(const char *stream, 
   111 					 ITextListener_SMC *smc,
   112 					 SMCStates *states)
   113 {
   114 	CharStream srdr = { stream };
   115 
   116 	return ParseStream_SMC(&srdr, CharStreamReader, smc, states);
   117 }
   118 
   119 /**
   120  * File streams
   121  */
   122 
   123 bool FileStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
   124 {
   125 	size_t num = fread(buffer, 1, maxlength, (FILE *)stream);
   126 
   127 	*read = static_cast<unsigned int>(num);
   128 
   129 	if (num == 0 && feof((FILE *)stream))
   130 	{
   131 		return true;
   132 	}
   133 
   134 	return (ferror((FILE *)stream) == 0);
   135 }
   136 
   137 SMCError TextParsers::ParseFile_SMC(const char *file, ITextListener_SMC *smc, SMCStates *states)
   138 {
   139 	FILE *fp = fopen(file, "rt");
   140 
   141 	if (!fp)
   142 	{
   143 		if (states != NULL)
   144 		{
   145 			states->line = 0;
   146 			states->col = 0;
   147 		}
   148 		return SMCError_StreamOpen;
   149 	}
   150 
   151 	SMCError result = ParseStream_SMC(fp, FileStreamReader, smc, states);
   152 
   153 	fclose(fp);
   154 
   155 	return result;
   156 }
   157 
   158 /** 
   159  * Raw parsing of streams with helper functions
   160  */
   161 
   162 struct StringInfo
   163 {
   164 	StringInfo() : quoted(false), ptr(NULL), end(NULL), special(false) { }
   165 	bool quoted;
   166 	char *ptr;
   167 	char *end;
   168 	bool special;
   169 };
   170 
   171 const char *FixupString(StringInfo &data)
   172 {
   173 	if (!data.ptr)
   174 	{
   175 		return NULL;
   176 	}
   177 
   178 	if (data.quoted)
   179 	{
   180 		data.ptr++;
   181 	}
   182 #if defined _DEBUG
   183 	else {
   184 		/* A string will never have beginning whitespace because we ignore it in the stream.
   185 		 * Furthermore, if there is trailing whitespace, the end ptr will point to it, so it is valid
   186 		 * to overwrite!  Lastly, the last character must be whitespace or a comment/invalid character.
   187 		 */
   188  	}
   189 #endif
   190 
   191 	/* Do some extra work on strings that have special quoted characters. */
   192 	if (data.special)
   193 	{
   194 		char *outptr = data.ptr;
   195 		size_t len = data.end - data.ptr;
   196 		if (len >= 2)
   197 		{
   198 			for (size_t i=0; i<len; i++)
   199 			{
   200 				if (data.ptr[i] == '\\' && i < len - 1)
   201 				{
   202 					/* Resolve the next character. */
   203 					i++;
   204 					if (data.ptr[i] == 'n')
   205 					{
   206 						data.ptr[i] = '\n';
   207 					} else if (data.ptr[i] == 't') {
   208 						data.ptr[i] = '\t';
   209 					} else if (data.ptr[i] == 'r') {
   210 						data.ptr[i] = '\r';
   211 					} else if (data.ptr[i] != '\\'
   212 						&& data.ptr[i] != '"') {
   213 						/* This character is invalid, so go back one */
   214 						i--;
   215 					}
   216 				}
   217 				*outptr++ = data.ptr[i];
   218 			}
   219 			*outptr = '\0';
   220 		}
   221 	}
   222 
   223 	if (data.end)
   224 	{
   225 		*(data.end) = '\0';
   226 	}
   227 
   228 	return data.ptr;
   229 }
   230 
   231 const char *rotate(StringInfo info[3])
   232 {
   233 	if (info[2].ptr != NULL)
   234 	{
   235 		return info[2].ptr;
   236 	}
   237 
   238 	if (info[0].ptr != NULL)
   239 	{
   240 		info[2] = info[1];
   241 		info[1] = info[0];
   242 		info[0] = StringInfo();
   243 	}
   244 	
   245 	return NULL;
   246 }
   247 
   248 void scrap(StringInfo info[3])
   249 {
   250 	info[2] = StringInfo();
   251 	info[1] = StringInfo();
   252 	info[0] = StringInfo();
   253 }
   254 
   255 void reloc(StringInfo &data, unsigned int bytes)
   256 {
   257 	if (data.ptr)
   258 	{
   259 		data.ptr -= bytes;
   260 	}
   261 	if (data.end)
   262 	{
   263 		data.end -= bytes;
   264 	}
   265 }
   266 
   267 char *lowstring(StringInfo info[3])
   268 {
   269 	for (int i=2; i>=0; i--)
   270 	{
   271 		if (info[i].ptr)
   272 		{
   273 			return info[i].ptr;
   274 		}
   275 	}
   276 
   277 	return NULL;
   278 }
   279 
   280 SMCError TextParsers::ParseStream_SMC(void *stream, 
   281 								   STREAMREADER srdr, 
   282 								   ITextListener_SMC *smc, 
   283 								   SMCStates *pStates)
   284 {
   285 	char *reparse_point = NULL;
   286 	char in_buf[4096];
   287 	char *parse_point = in_buf;
   288 	char *line_begin = in_buf;
   289 	unsigned int read;
   290 	unsigned int curlevel = 0;
   291 	bool in_quote = false;
   292 	bool ignoring = false;
   293 	bool eol_comment = false;
   294 	bool ml_comment = false;
   295 	unsigned int i;
   296 	SMCError err = SMCError_Okay;
   297 	SMCResult res;
   298 	SMCStates states;
   299 	char c;
   300 
   301 	StringInfo strings[3];
   302 	StringInfo emptystring;
   303 
   304 	states.line = 1;
   305 	states.col = 0;
   306 
   307 	smc->ReadSMC_ParseStart();
   308 
   309 	/**
   310 	 * The stream reader reads in as much as it can fill the buffer with.
   311 	 * It then processes the buffer.  If the buffer cannot be fully processed, for example, 
   312 	 * a line is left hanging with no newline, then the contents of the buffer is shifted 
   313 	 * down, and the buffer is filled from the stream reader again.
   314 	 *
   315 	 * What makes this particularly annoying is that we cache pointers everywhere, so when 
   316 	 * the shifting process takes place, all those pointers must be shifted as well.
   317 	 */
   318 	while (srdr(stream, parse_point, sizeof(in_buf) - (parse_point - in_buf) - 1, &read))
   319 	{
   320 		if (!read)
   321 		{
   322 			break;
   323 		}
   324 
   325 		/* Check for BOM markings, which is only relevant on the first line.
   326 		 * Not worth it, but it could be moved out of the loop.
   327 		 */
   328 		if (states.line == 1 && 
   329 			in_buf[0] == (char)0xEF && 
   330 			in_buf[1] == (char)0xBB && 
   331 			in_buf[2] == (char)0xBF)
   332 		{
   333 			/* Move EVERYTHING down :\ */
   334 			memmove(in_buf, &in_buf[3], read - 3);
   335 			read -= 3;
   336 		}
   337 
   338 		if (reparse_point)
   339 		{
   340 			read += (parse_point - reparse_point);
   341 			parse_point = reparse_point;
   342 			reparse_point = NULL;
   343 		}
   344 
   345 		for (i=0; i<read; i++)
   346 		{
   347 			c = parse_point[i];
   348 			if (c == '\n')
   349 			{
   350 				/* If we got a newline, there's a lot of things that could have happened in the interim.
   351 				 * First, let's make sure the staged strings are rotated.
   352 				 */
   353 				if (strings[0].ptr)
   354 				{
   355 					strings[0].end = &parse_point[i];
   356 					if (rotate(strings) != NULL)
   357 					{
   358 						err = SMCError_InvalidTokens;
   359 						goto failed;
   360 					}
   361 				}
   362 
   363 				/* Next, let's clear some line-based values that may no longer have meaning */
   364 				eol_comment = false;
   365 				in_quote = false;
   366 				if (ignoring && !ml_comment)
   367 				{
   368 					ignoring = false;
   369 				}
   370 
   371 				/* Pass the raw line onto the listener.  We terminate the line so the receiver 
   372 				 * doesn't get tons of useless info.  We restore the newline after.
   373 				 */
   374 				parse_point[i] = '\0';
   375 				if ((res=smc->ReadSMC_RawLine(&states, line_begin)) != SMCResult_Continue)
   376 				{
   377 					err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
   378 					goto failed;
   379 				}
   380 				parse_point[i] = '\n';
   381 
   382 				/* Now we check the sanity of our staged strings! */
   383 				if (strings[2].ptr)
   384 				{
   385 					if (!curlevel)
   386 					{
   387 						err = SMCError_InvalidProperty1;
   388 						goto failed;
   389 					}
   390 					/* Assume the next string is a property and pass the info on. */
   391 					if ((res=smc->ReadSMC_KeyValue(
   392 						&states,
   393 						FixupString(strings[2]),
   394 						FixupString(strings[1]))) != SMCResult_Continue)
   395 					{
   396 						err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
   397 						goto failed;
   398 					}
   399 					scrap(strings);
   400 				}
   401 
   402 				/* Change the states for the next line */
   403 				states.col = 0;
   404 				states.line++;
   405 				line_begin = &parse_point[i+1];		//Note: safe because this gets relocated later
   406 			} 
   407 			else if (ignoring) 
   408 			{
   409 				if (in_quote)
   410 				{
   411 					/* If i was 0, we could have reparsed, so make sure there's no buffer underrun */
   412 					if ((&parse_point[i] != in_buf) && c == '"' && parse_point[i-1] != '\\')
   413 					{
   414 						/* If we reached a quote in an ignore phase,
   415 						 * we're staging a string and we must rotate it out.
   416 						 */
   417 						in_quote = false;
   418 						ignoring = false;
   419 						/* Set our info */
   420 						strings[0].end = &parse_point[i];
   421 						strings[0].quoted = true;
   422 						if (rotate(strings) != NULL)
   423 						{
   424 							/* If we rotated too many strings, there was too much crap on one line */
   425 							err = SMCError_InvalidTokens;
   426 							goto failed;
   427 						}
   428 					} 
   429 					else if (c == '\\') 
   430 					{
   431 						strings[0].special = true;
   432 						if (i == (read - 1))
   433 						{
   434 							reparse_point = &parse_point[i];
   435 							break;
   436 						}
   437 					}
   438 				} 
   439 				else if (ml_comment) 
   440 				{
   441 					if (c == '*')
   442 					{
   443 						/* Check if we need to get more input first */
   444 						if (i == read - 1)
   445 						{
   446 							reparse_point = &parse_point[i];
   447 							break;
   448 						}
   449 						if (parse_point[i+1] == '/')
   450 						{
   451 							ml_comment = false;
   452 							ignoring = false;
   453 							/* We should not be staging anything right now. */
   454 							assert(strings[0].ptr == NULL);
   455 							/* Advance the input stream so we don't choke on this token */
   456 							i++;
   457 							states.col++;
   458 						}
   459 					}
   460 				}
   461 			} 
   462 			else 
   463 			{
   464 				/* Check if we're whitespace or not */
   465 				if (!g_ws_chartable[(unsigned)c])
   466 				{
   467 					bool restage = false;
   468 					/* Check various special tokens:
   469 					 * ;
   470 					 * //
   471 					 * / *
   472 					 * {
   473 					 * }
   474 					 */
   475 					if (c == ';' || c == '/')
   476 					{
   477 						/* If it's a line-based comment (that is, ; or //)
   478 						 * we will need to scrap everything until the end of the line.
   479 						 */
   480 						if (c == '/')
   481 						{
   482 							if (i == read - 1)
   483 							{
   484 								/* If we reached the end of the look-ahead, we need to re-check our input.
   485 								 * Breaking out will force this to be the new reparse point!
   486 								 */
   487 								reparse_point = &parse_point[i];
   488 								break;
   489 							}
   490 							if (parse_point[i + 1] == '/')
   491 							{
   492 								/* standard comment */
   493 								ignoring = true;
   494 								eol_comment = true;
   495 								restage = true;
   496 							} 
   497 							else if (parse_point[i+1] == '*') 
   498 							{
   499 								/* inline comment - start ignoring */
   500 								ignoring = true;
   501 								ml_comment = true;
   502 								/* yes, we restage, meaning that:
   503 								 * STR/ *stuff* /ING  (space because ml comments don't nest in C++)
   504 								 * will not generate 'STRING', but rather 'STR' and 'ING'.
   505 								 * This should be a rare occurrence and is done here for convenience.
   506 								 */
   507 								restage = true;
   508 							}
   509 						} 
   510 						else 
   511 						{
   512 							ignoring = true;
   513 							eol_comment = true;
   514 							restage = true;
   515 						}
   516 					} 
   517 					else if (c == '{') 
   518 					{
   519 						/* If we are staging a string, we must rotate here */
   520 						if (strings[0].ptr)
   521 						{
   522 							/* We have unacceptable tokens on this line */
   523 							if (rotate(strings) != NULL)
   524 							{
   525 								err = SMCError_InvalidSection1;
   526 								goto failed;
   527 							}
   528 						}
   529 						/* Sections must always be alone */
   530 						if (strings[2].ptr != NULL)
   531 						{
   532 							err = SMCError_InvalidSection1;
   533 							goto failed;
   534 						} 
   535 						else if (strings[1].ptr == NULL)
   536 						{
   537 							err = SMCError_InvalidSection2;
   538 							goto failed;
   539 						}
   540 						if ((res=smc->ReadSMC_NewSection(&states, FixupString(strings[1])))
   541 							!= SMCResult_Continue)
   542 						{
   543 							err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
   544 							goto failed;
   545 						}
   546 						strings[1] = emptystring;
   547 						curlevel++;
   548 					} 
   549 					else if (c == '}') 
   550 					{
   551 						/* Unlike our matching friend, this can be on the same line as something prior */
   552 						if (rotate(strings) != NULL)
   553 						{
   554 							err = SMCError_InvalidSection3;
   555 							goto failed;
   556 						}
   557 						if (strings[2].ptr)
   558 						{
   559 							if (!curlevel)
   560 							{
   561 								err = SMCError_InvalidProperty1;
   562 								goto failed;
   563 							}
   564 							if ((res=smc->ReadSMC_KeyValue(
   565 											&states,
   566 											FixupString(strings[2]),
   567 											FixupString(strings[1])))
   568 								!= SMCResult_Continue)
   569 							{
   570 								err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
   571 								goto failed;
   572 							}
   573 						} 
   574 						else if (strings[1].ptr) 
   575 						{
   576 							err = SMCError_InvalidSection3;
   577 							goto failed;
   578 						} 
   579 						else if (!curlevel) 
   580 						{
   581 							err = SMCError_InvalidSection4;
   582 							goto failed;
   583 						}
   584 						/* Now it's safe to leave the section */
   585 						scrap(strings);
   586 						if ((res=smc->ReadSMC_LeavingSection(&states)) != SMCResult_Continue)
   587 						{
   588 							err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
   589 							goto failed;
   590 						}
   591 						curlevel--;
   592 					} 
   593 					else if (c == '"') 
   594 					{
   595 						/* If we get a quote mark, we always restage, but we need to do it beforehand */
   596 						if (strings[0].ptr)
   597 						{
   598 							strings[0].end = &parse_point[i];
   599 							if (rotate(strings) != NULL)
   600 							{
   601 								err = SMCError_InvalidTokens;
   602 								goto failed;
   603 							}
   604 						}
   605 						strings[0].ptr = &parse_point[i];
   606 						in_quote = true;
   607 						ignoring = true;
   608 					} 
   609 					else if (!strings[0].ptr) 
   610 					{
   611 						/* If we have no string, we must start one */
   612 						strings[0].ptr = &parse_point[i];
   613 					}
   614 					if (restage && strings[0].ptr)
   615 					{
   616 						strings[0].end = &parse_point[i];
   617 						if (rotate(strings) != NULL)
   618 						{
   619 							err = SMCError_InvalidTokens;
   620 							goto failed;
   621 						}
   622 					}
   623 				} 
   624 				else 
   625 				{
   626 					/* If we're eating a string and get whitespace, we need to restage.
   627 					 * (Note that if we are quoted, this is being ignored)
   628 					 */
   629 					if (strings[0].ptr)
   630 					{
   631 						/*
   632 						 * The specification says the second string in a pair does not need to be quoted.
   633 						 * Thus, we check if there's already a string on the stack.
   634 						 * If there's a newline, we always rotate so the newline has an empty starter.
   635 						 */
   636 						if (!strings[1].ptr)
   637 						{
   638 							/* There's no string, so we must move this one down and eat up another */
   639 							strings[0].end = &parse_point[i];
   640 							rotate(strings);
   641 						} 
   642 						else if (!strings[1].quoted) 
   643 						{
   644 							err = SMCError_InvalidTokens;
   645 							goto failed;
   646 						}
   647 					}
   648 				}
   649 			}
   650 
   651 			/* Advance which token we're on */
   652 			states.col++;
   653 		}
   654 
   655 		if (line_begin != in_buf)
   656 		{
   657 			/* The line buffer has advanced, so it's safe to copy N bytes back to the beginning.
   658 			 * What's N?  N is the lowest point we're currently relying on.
   659 			 */
   660 			char *stage = lowstring(strings);
   661 			if (!stage || stage > line_begin)
   662 			{
   663 				stage = line_begin;
   664 			}
   665 			unsigned int bytes = read - (stage - parse_point);
   666 
   667 			/* It is now safe to delete everything before the staged point */
   668 			memmove(in_buf, stage, bytes);
   669 
   670 			/* Calculate the number of bytes in the new buffer */
   671 			bytes = stage - in_buf;
   672 			/* Relocate all the cached pointers to our new base */
   673 			line_begin -= bytes;
   674 			reloc(strings[0], bytes);
   675 			reloc(strings[1], bytes);
   676 			reloc(strings[2], bytes);
   677 			if (reparse_point)
   678 			{
   679 				reparse_point -= bytes;
   680 			}
   681 			if (parse_point)
   682 			{
   683 				parse_point = &parse_point[read];
   684 				parse_point -= bytes;
   685 			}
   686 		} 
   687 		else if (read == sizeof(in_buf) - 1) 
   688 		{
   689 			err = SMCError_TokenOverflow;
   690 			goto failed;
   691 		}
   692 	}
   693 
   694 	/* If we're done parsing and there are tokens left over... */
   695 	if (curlevel)
   696 	{
   697 		err = SMCError_InvalidSection5;
   698 		goto failed;
   699 	} 
   700 	else if (strings[0].ptr || strings[1].ptr) 
   701 	{
   702 		err = SMCError_InvalidTokens;
   703 		goto failed;
   704 	}
   705 	
   706 	smc->ReadSMC_ParseEnd(false, false);
   707 	
   708 	if (pStates != NULL)
   709 	{
   710 		*pStates = states;
   711 	}
   712 
   713 	return SMCError_Okay;
   714 
   715 failed:
   716 	if (pStates != NULL)
   717 	{
   718 		*pStates = states;
   719 	}
   720 
   721 	smc->ReadSMC_ParseEnd(true, (err == SMCError_Custom));
   722 
   723 	return err;
   724 }
   725 
   726 
   727 /**
   728  * INI parser 
   729  */
   730 
   731 bool TextParsers::ParseFile_INI(const char *file, ITextListener_INI *ini_listener, unsigned int *line, unsigned int *col)
   732 {
   733 	FILE *fp = fopen(file, "rt");
   734 	unsigned int curline = 0;
   735 	unsigned int curtok;
   736 	size_t len;
   737 
   738 	if (!fp)
   739 	{
   740 		if (line)
   741 		{
   742 			*line = 0;
   743 		}
   744 
   745 		return false;
   746 	}
   747 
   748 	char buffer[2048];
   749 	char *ptr, *save_ptr;
   750 	bool in_quote;
   751 
   752 	while (!feof(fp))
   753 	{
   754 		curline++;
   755 		curtok = 0;
   756 		buffer[0] = '\0';
   757 		if (fgets(buffer, sizeof(buffer), fp) == NULL)
   758 		{
   759 			break;
   760 		}
   761 
   762 		//:TODO: this will only run once, so find a nice way to move it out of the while loop
   763 		/* If this is the first line, check the first three bytes for BOM */
   764 		if (curline == 1 && 
   765 			buffer[0] == (char)0xEF && 
   766 			buffer[1] == (char)0xBB && 
   767 			buffer[2] == (char)0xBF)
   768 		{
   769 			/* We have a UTF-8 marked file... skip these bytes */
   770 			ptr = &buffer[3];
   771 		} else {
   772 			ptr = buffer;
   773 		}
   774 
   775 		/***************************************************
   776 		 * We preprocess the string before parsing tokens! *
   777 		 ***************************************************/
   778 
   779 		/* First strip beginning whitespace */
   780 		while (*ptr != '\0' && g_ws_chartable[(unsigned)*ptr] != 0)
   781 		{
   782 			ptr++;
   783 		}
   784 
   785 		len = strlen(ptr);
   786 
   787 		if (!len)
   788 		{
   789 			continue;
   790 		}
   791 
   792 		/* Now search for comment characters */
   793 		in_quote = false;
   794 		save_ptr = ptr;
   795 		for (size_t i=0; i<len; i++,ptr++)
   796 		{
   797 			if (!in_quote)
   798 			{
   799 				switch (*ptr)
   800 				{
   801 				case '"':
   802 					{
   803 						in_quote = true;
   804 						break;
   805 					}
   806 				case ';':
   807 					{
   808 						/* Stop the loop */
   809 						len = i;
   810 						/* Terminate the string here */
   811 						*ptr = '\0';
   812 						break;
   813 					}
   814 				}
   815 			} else {
   816 				if (*ptr == '"')
   817 				{
   818 					in_quote = false;
   819 				}
   820 			}
   821 		}
   822 
   823 		if (!len)
   824 		{
   825 			continue;
   826 		}
   827 
   828 		ptr = save_ptr;
   829 
   830 		/* Lastly, strip ending whitespace off */
   831 		for (size_t i=len-1; i>=0 && i<len; i--)
   832 		{
   833 			if (g_ws_chartable[(unsigned)ptr[i]])
   834 			{
   835 				ptr[i] = '\0';
   836 				len--;
   837 			} else {
   838 				break;
   839 			}
   840 		}
   841 
   842 		if (!len)
   843 		{
   844 			continue;
   845 		}
   846 
   847 		if (!ini_listener->ReadINI_RawLine(ptr, &curtok))
   848 		{
   849 			goto event_failed;
   850 		}
   851 
   852 		if (*ptr == '[')
   853 		{
   854 			bool invalid_tokens = false;
   855 			bool got_bracket = false;
   856 			bool extra_tokens = false;
   857 			char c;
   858 			bool alnum;
   859 			wchar_t wc;
   860 
   861 			for (size_t i=1; i<len; i++)
   862 			{
   863 				c = ptr[i];
   864 				alnum = false;
   865 
   866 				if (c & (1<<7))
   867 				{
   868 					if (mbtowc(&wc, &ptr[i], len-i) != -1)
   869 					{
   870 						alnum = (iswalnum(wc) != 0);
   871 						i += _GetUTF8CharBytes(&ptr[i]) - 1;
   872 					}
   873 				} else {
   874 					alnum = (isalnum(c) != 0) || (g_ini_chartable1[(unsigned)c] != 0);
   875 				}
   876 				if (!alnum)
   877 				{
   878 					/* First check - is this a bracket? */
   879 					if (c == ']')
   880 					{
   881 						/* Yes! */
   882 						got_bracket = true;
   883 						/* If this isn't the last character... */
   884 						if (i != len - 1)
   885 						{
   886 							extra_tokens = true;
   887 						}
   888 						/* terminate */
   889 						ptr[i] = '\0';
   890 						break;
   891 					} else {
   892 						/* n...No! Continue copying. */
   893 						invalid_tokens = true;
   894 					}
   895 				}
   896 			}
   897 
   898 			/* Tell the handler */
   899 			if (!ini_listener->ReadINI_NewSection(&ptr[1], invalid_tokens, got_bracket, extra_tokens, &curtok))
   900 			{
   901 				goto event_failed;
   902 			}
   903 		} else {
   904 			char *key_ptr = ptr;
   905 			char *val_ptr = NULL;
   906 			char c;
   907 			size_t first_space = 0;
   908 			bool invalid_tokens = false;
   909 			bool equal_token = false;
   910 			bool quotes = false;
   911 			bool alnum;
   912 			wchar_t wc;
   913 
   914 			for (size_t i=0; i<len; i++)
   915 			{
   916 				c = ptr[i];
   917 				alnum = false;
   918 				/* is this an invalid char? */
   919 				if (c & (1<<7))
   920 				{
   921 					if (mbtowc(&wc, &ptr[i], len-i) != -1)
   922 					{
   923 						alnum = (iswalnum(wc) != 0);
   924 						i += _GetUTF8CharBytes(&ptr[i]) - 1;
   925 					}
   926 				} else {
   927 					alnum = (isalnum(c) != 0) || (g_ini_chartable1[(unsigned)c] != 0);
   928 				}
   929 
   930 				if (!alnum)
   931 				{
   932 					if (g_ws_chartable[(unsigned)c])
   933 					{
   934 						/* if it's a space, keep track of the first occurring space */
   935 						if (!first_space)
   936 						{
   937 							first_space = i;
   938 						}
   939 					} else {
   940 						if (c == '=')
   941 						{
   942 							/* if it's an equal sign, we're done with the key */
   943 							if (first_space)
   944 							{
   945 								/* remove excess whitespace */
   946 								key_ptr[first_space] = '\0';
   947 							} else {
   948 								/* remove the equal sign */
   949 								key_ptr[i] = '\0';
   950 							}
   951 							if (ptr[++i] != '\0')
   952 							{
   953 								/* If this isn't the end, set next pointer */
   954 								val_ptr = &ptr[i];
   955 							}
   956 							equal_token = true;
   957 							break;
   958 						} else {
   959 							/* Mark that we got something invalid! */
   960 							invalid_tokens = true;
   961 							first_space = 0;
   962 						}
   963 					}
   964 				}
   965 			}
   966 
   967 			/* Now we need to parse the value, if any */
   968 			if (val_ptr)
   969 			{
   970 				/* eat up spaces! there shouldn't be any h*/
   971 				while ((*val_ptr != '\0') && g_ws_chartable[(unsigned)*val_ptr] != 0)
   972 				{
   973 					val_ptr++;
   974 				}
   975 				if (*val_ptr == '\0')
   976 				{
   977 					val_ptr = NULL;
   978 					goto skip_value;
   979 				}
   980 				/* Do we have an initial quote? If so, the parsing rules change! */
   981 				if (*val_ptr == '"' && *val_ptr != '\0')
   982 				{
   983 					len = strlen(val_ptr);
   984 					if (val_ptr[len-1] == '"')
   985 					{
   986 						/* Strip quotes! */
   987 						val_ptr[--len] = '\0';
   988 						val_ptr++;
   989 						quotes = true;
   990 					}
   991 				}
   992 			}
   993 skip_value:
   994 			/* We're done! */
   995 			curtok = val_ptr - buffer;
   996 			if (!ini_listener->ReadINI_KeyValue(key_ptr, val_ptr, invalid_tokens, equal_token, quotes, &curtok))
   997 			{
   998 				curtok = 0;
   999 				goto event_failed;
  1000 			}
  1001 		}
  1002 	}
  1003 
  1004 	if (line)
  1005 	{
  1006 		*line = curline;
  1007 	}
  1008 
  1009 	fclose(fp);
  1010 
  1011 	return true;
  1012 
  1013 event_failed:
  1014 	if (line)
  1015 	{
  1016 		*line = curline;
  1017 	}
  1018 
  1019 	if (col)
  1020 	{
  1021 		*col = curtok;
  1022 	}
  1023 
  1024 	fclose(fp);
  1025 
  1026 	return false;
  1027 }
  1028 
  1029 const char *TextParsers::GetSMCErrorString(SMCError err)
  1030 {
  1031 	static const char *s_errors[] = 
  1032 	{
  1033 		NULL,
  1034 		"Stream failed to open",
  1035 		"Stream returned read error",
  1036 		NULL,
  1037 		"Un-quoted section has invalid tokens",
  1038 		"Section declared without header",
  1039 		"Section declared with unknown tokens",
  1040 		"Section ending without a matching section beginning",
  1041 		"Section beginning without a matching ending",
  1042 		"Line contained too many invalid tokens",
  1043 		"Token buffer overflowed",
  1044 		"A property was declared outside of a section",
  1045 	};
  1046 
  1047 	if (err < SMCError_Okay || err > SMCError_InvalidProperty1)
  1048 	{
  1049 		return NULL;
  1050 	}
  1051 
  1052 	return s_errors[err];
  1053 }