core/TextParsers.cpp
author Michael McKoy <ferret99gt@gmail.com>
Sat Nov 24 03:28:33 2007 +0000 (2007-11-24)
changeset 1719 c9d5e55f0899
parent 1656 b0b86d57530c
child 1730 62dd251bb1cd
permissions -rw-r--r--
The current map played shouldn't appear in MapChooser votes or RTV votes now. It can still be nominated in RTV though.
dvander@591
     1
/**
ds@653
     2
 * vim: set ts=4 :
ds@1330
     3
 * =============================================================================
dvander@1231
     4
 * SourceMod
dvander@1231
     5
 * Copyright (C) 2004-2007 AlliedModders LLC.  All rights reserved.
ds@1330
     6
 * =============================================================================
dvander@591
     7
 *
ds@1330
     8
 * This program is free software; you can redistribute it and/or modify it under
ds@1330
     9
 * the terms of the GNU General Public License, version 3.0, as published by the
ds@1330
    10
 * Free Software Foundation.
dvander@1231
    11
 * 
ds@1330
    12
 * This program is distributed in the hope that it will be useful, but WITHOUT
ds@1330
    13
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
ds@1330
    14
 * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
ds@1330
    15
 * details.
dvander@1231
    16
 *
ds@1330
    17
 * You should have received a copy of the GNU General Public License along with
ds@1330
    18
 * this program.  If not, see <http://www.gnu.org/licenses/>.
dvander@1231
    19
 *
ds@1330
    20
 * As a special exception, AlliedModders LLC gives you permission to link the
ds@1330
    21
 * code of this program (as well as its derivative works) to "Half-Life 2," the
ds@1330
    22
 * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software
ds@1330
    23
 * by the Valve Corporation.  You must obey the GNU General Public License in
ds@1330
    24
 * all respects for all other code used.  Additionally, AlliedModders LLC grants
ds@1330
    25
 * this exception to all derivative works.  AlliedModders LLC defines further
ds@1330
    26
 * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),
ds@1330
    27
 * or <http://www.sourcemod.net/license.php>.
dvander@591
    28
 *
dvander@591
    29
 * Version: $Id$
dvander@591
    30
 */
dvander@591
    31
dvander@591
    32
#include <stdio.h>
dvander@591
    33
#include <ctype.h>
dvander@591
    34
#include <wctype.h>
dvander@591
    35
#include <string.h>
dvander@591
    36
#include <stdlib.h>
dvander@591
    37
#include <assert.h>
dvander@591
    38
#include "TextParsers.h"
dvander@591
    39
#include "ShareSys.h"
dvander@591
    40
dvander@616
    41
TextParsers g_TextParser;
dvander@1406
    42
ITextParsers *textparsers = &g_TextParser;
dvander@591
    43
dvander@591
    44
static int g_ini_chartable1[255] = {0};
dvander@591
    45
static int g_ws_chartable[255] = {0};
dvander@591
    46
dvander@1406
    47
bool TextParsers::IsWhitespace(const char *stream)
dvander@631
    48
{
dvander@631
    49
	return g_ws_chartable[(unsigned)*stream] == 1;
dvander@631
    50
}
dvander@631
    51
dvander@616
    52
TextParsers::TextParsers()
dvander@591
    53
{
dvander@591
    54
	g_ini_chartable1[(unsigned)'_'] = 1;
dvander@591
    55
	g_ini_chartable1[(unsigned)'-'] = 1;
dvander@591
    56
	g_ini_chartable1[(unsigned)','] = 1;
dvander@591
    57
	g_ini_chartable1[(unsigned)'+'] = 1;
dvander@591
    58
	g_ini_chartable1[(unsigned)'.'] = 1;
dvander@591
    59
	g_ini_chartable1[(unsigned)'$'] = 1;
dvander@591
    60
	g_ini_chartable1[(unsigned)'?'] = 1;
dvander@591
    61
	g_ini_chartable1[(unsigned)'/'] = 1;
dvander@591
    62
	g_ws_chartable[(unsigned)'\n'] = 1;
dvander@591
    63
	g_ws_chartable[(unsigned)'\v'] = 1;
dvander@591
    64
	g_ws_chartable[(unsigned)'\r'] = 1;
dvander@591
    65
	g_ws_chartable[(unsigned)'\t'] = 1;
dvander@591
    66
	g_ws_chartable[(unsigned)'\f'] = 1;
dvander@591
    67
	g_ws_chartable[(unsigned)' '] = 1;
dvander@591
    68
}
dvander@591
    69
dvander@616
    70
void TextParsers::OnSourceModAllInitialized()
dvander@591
    71
{
dvander@591
    72
	g_ShareSys.AddInterface(NULL, this);
dvander@591
    73
}
dvander@591
    74
dvander@616
    75
unsigned int TextParsers::GetUTF8CharBytes(const char *stream)
dvander@591
    76
{
dvander@591
    77
	return _GetUTF8CharBytes(stream);
dvander@591
    78
}
dvander@591
    79
dvander@591
    80
/**
dvander@591
    81
 * Character streams
dvander@591
    82
 */
dvander@591
    83
dvander@591
    84
struct CharStream
dvander@591
    85
{
dvander@591
    86
	const char *curpos;
dvander@591
    87
};
dvander@591
    88
dvander@591
    89
bool CharStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
dvander@591
    90
{
dvander@591
    91
	CharStream *srdr = (CharStream *)stream;
dvander@591
    92
dvander@591
    93
	const char *ptr = srdr->curpos;
dvander@591
    94
	for (size_t i=0; i<maxlength; i++)
dvander@591
    95
	{
dvander@591
    96
		if (*ptr == '\0')
dvander@591
    97
		{
dvander@591
    98
			break;
dvander@591
    99
		}
dvander@591
   100
		*buffer++ = *ptr++;
dvander@591
   101
	}
dvander@591
   102
dvander@591
   103
	*read = ptr - srdr->curpos;
dvander@591
   104
dvander@591
   105
	srdr->curpos = ptr;
dvander@591
   106
dvander@591
   107
	return true;
dvander@591
   108
}
dvander@591
   109
dvander@1656
   110
SMCError TextParsers::ParseString_SMC(const char *stream, 
dvander@591
   111
					 ITextListener_SMC *smc,
dvander@1656
   112
					 SMCStates *states)
dvander@591
   113
{
dvander@591
   114
	CharStream srdr = { stream };
dvander@591
   115
dvander@1656
   116
	return ParseStream_SMC(&srdr, CharStreamReader, smc, states);
dvander@591
   117
}
dvander@591
   118
dvander@591
   119
/**
dvander@591
   120
 * File streams
dvander@591
   121
 */
dvander@591
   122
dvander@591
   123
bool FileStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
dvander@591
   124
{
dvander@591
   125
	size_t num = fread(buffer, 1, maxlength, (FILE *)stream);
dvander@591
   126
dvander@591
   127
	*read = static_cast<unsigned int>(num);
dvander@591
   128
dvander@591
   129
	if (num == 0 && feof((FILE *)stream))
dvander@591
   130
	{
dvander@591
   131
		return true;
dvander@591
   132
	}
dvander@591
   133
dvander@591
   134
	return (ferror((FILE *)stream) == 0);
dvander@591
   135
}
dvander@591
   136
dvander@1656
   137
SMCError TextParsers::ParseFile_SMC(const char *file, ITextListener_SMC *smc, SMCStates *states)
dvander@591
   138
{
dvander@591
   139
	FILE *fp = fopen(file, "rt");
dvander@591
   140
dvander@591
   141
	if (!fp)
dvander@591
   142
	{
dvander@1656
   143
		if (states != NULL)
dvander@1656
   144
		{
dvander@1656
   145
			states->line = 0;
dvander@1656
   146
			states->col = 0;
dvander@1656
   147
		}
dvander@1656
   148
		return SMCError_StreamOpen;
dvander@591
   149
	}
dvander@591
   150
dvander@1656
   151
	SMCError result = ParseStream_SMC(fp, FileStreamReader, smc, states);
dvander@591
   152
dvander@591
   153
	fclose(fp);
dvander@591
   154
dvander@591
   155
	return result;
dvander@591
   156
}
dvander@591
   157
dvander@591
   158
/** 
dvander@591
   159
 * Raw parsing of streams with helper functions
dvander@591
   160
 */
dvander@591
   161
dvander@591
   162
struct StringInfo
dvander@591
   163
{
dvander@591
   164
	StringInfo() : quoted(false), ptr(NULL), end(NULL), special(false) { }
dvander@591
   165
	bool quoted;
dvander@591
   166
	char *ptr;
dvander@591
   167
	char *end;
dvander@591
   168
	bool special;
dvander@591
   169
};
dvander@591
   170
dvander@591
   171
const char *FixupString(StringInfo &data)
dvander@591
   172
{
dvander@591
   173
	if (!data.ptr)
dvander@591
   174
	{
dvander@591
   175
		return NULL;
dvander@591
   176
	}
dvander@591
   177
dvander@591
   178
	if (data.quoted)
dvander@591
   179
	{
dvander@591
   180
		data.ptr++;
dvander@591
   181
	}
dvander@591
   182
#if defined _DEBUG
dvander@591
   183
	else {
dvander@591
   184
		/* A string will never have beginning whitespace because we ignore it in the stream.
dvander@591
   185
		 * Furthermore, if there is trailing whitespace, the end ptr will point to it, so it is valid
dvander@591
   186
		 * to overwrite!  Lastly, the last character must be whitespace or a comment/invalid character.
dvander@591
   187
		 */
dvander@591
   188
 	}
dvander@591
   189
#endif
dvander@591
   190
dvander@1073
   191
	/* Do some extra work on strings that have special quoted characters. */
dvander@591
   192
	if (data.special)
dvander@591
   193
	{
dvander@1073
   194
		char *outptr = data.ptr;
dvander@1073
   195
		size_t len = data.end - data.ptr;
dvander@1073
   196
		if (len >= 2)
dvander@1073
   197
		{
dvander@1073
   198
			for (size_t i=0; i<len; i++)
dvander@1073
   199
			{
dvander@1073
   200
				if (data.ptr[i] == '\\' && i < len - 1)
dvander@1073
   201
				{
dvander@1073
   202
					/* Resolve the next character. */
dvander@1073
   203
					i++;
dvander@1073
   204
					if (data.ptr[i] == 'n')
dvander@1073
   205
					{
dvander@1073
   206
						data.ptr[i] = '\n';
dvander@1073
   207
					} else if (data.ptr[i] == 't') {
dvander@1073
   208
						data.ptr[i] = '\t';
dvander@1073
   209
					} else if (data.ptr[i] == 'r') {
dvander@1073
   210
						data.ptr[i] = '\r';
dvander@1073
   211
					} else if (data.ptr[i] != '\\'
dvander@1073
   212
						&& data.ptr[i] != '"') {
dvander@1073
   213
						/* This character is invalid, so go back one */
dvander@1073
   214
						i--;
dvander@1073
   215
					}
dvander@1073
   216
				}
dvander@1073
   217
				*outptr++ = data.ptr[i];
dvander@1073
   218
			}
dvander@1073
   219
			*outptr = '\0';
dvander@1073
   220
		}
dvander@591
   221
	}
dvander@591
   222
dvander@1260
   223
	if (data.end)
dvander@1260
   224
	{
dvander@1260
   225
		*(data.end) = '\0';
dvander@1260
   226
	}
dvander@591
   227
dvander@591
   228
	return data.ptr;
dvander@591
   229
}
dvander@591
   230
dvander@591
   231
const char *rotate(StringInfo info[3])
dvander@591
   232
{
dvander@591
   233
	if (info[2].ptr != NULL)
dvander@591
   234
	{
dvander@591
   235
		return info[2].ptr;
dvander@591
   236
	}
dvander@591
   237
dvander@591
   238
	if (info[0].ptr != NULL)
dvander@591
   239
	{
dvander@591
   240
		info[2] = info[1];
dvander@591
   241
		info[1] = info[0];
dvander@591
   242
		info[0] = StringInfo();
dvander@591
   243
	}
dvander@591
   244
	
dvander@591
   245
	return NULL;
dvander@591
   246
}
dvander@591
   247
dvander@591
   248
void scrap(StringInfo info[3])
dvander@591
   249
{
dvander@591
   250
	info[2] = StringInfo();
dvander@591
   251
	info[1] = StringInfo();
dvander@591
   252
	info[0] = StringInfo();
dvander@591
   253
}
dvander@591
   254
dvander@591
   255
void reloc(StringInfo &data, unsigned int bytes)
dvander@591
   256
{
dvander@591
   257
	if (data.ptr)
dvander@591
   258
	{
dvander@591
   259
		data.ptr -= bytes;
dvander@591
   260
	}
dvander@591
   261
	if (data.end)
dvander@591
   262
	{
dvander@591
   263
		data.end -= bytes;
dvander@591
   264
	}
dvander@591
   265
}
dvander@591
   266
dvander@591
   267
char *lowstring(StringInfo info[3])
dvander@591
   268
{
dvander@591
   269
	for (int i=2; i>=0; i--)
dvander@591
   270
	{
dvander@591
   271
		if (info[i].ptr)
dvander@591
   272
		{
dvander@591
   273
			return info[i].ptr;
dvander@591
   274
		}
dvander@591
   275
	}
dvander@591
   276
dvander@591
   277
	return NULL;
dvander@591
   278
}
dvander@591
   279
dvander@1656
   280
SMCError TextParsers::ParseStream_SMC(void *stream, 
dvander@591
   281
								   STREAMREADER srdr, 
dvander@591
   282
								   ITextListener_SMC *smc, 
dvander@1656
   283
								   SMCStates *pStates)
dvander@591
   284
{
dvander@1503
   285
	char *reparse_point = NULL;
dvander@591
   286
	char in_buf[4096];
dvander@591
   287
	char *parse_point = in_buf;
dvander@591
   288
	char *line_begin = in_buf;
dvander@591
   289
	unsigned int read;
dvander@591
   290
	unsigned int curlevel = 0;
dvander@591
   291
	bool in_quote = false;
dvander@591
   292
	bool ignoring = false;
dvander@591
   293
	bool eol_comment = false;
dvander@591
   294
	bool ml_comment = false;
dvander@591
   295
	unsigned int i;
dvander@1656
   296
	SMCError err = SMCError_Okay;
dvander@1656
   297
	SMCResult res;
dvander@1656
   298
	SMCStates states;
dvander@591
   299
	char c;
dvander@591
   300
dvander@591
   301
	StringInfo strings[3];
dvander@591
   302
	StringInfo emptystring;
dvander@591
   303
dvander@1656
   304
	states.line = 1;
dvander@1656
   305
	states.col = 0;
dvander@1656
   306
dvander@591
   307
	smc->ReadSMC_ParseStart();
dvander@591
   308
dvander@1656
   309
	/**
dvander@1656
   310
	 * The stream reader reads in as much as it can fill the buffer with.
dvander@1656
   311
	 * It then processes the buffer.  If the buffer cannot be fully processed, for example, 
dvander@1656
   312
	 * a line is left hanging with no newline, then the contents of the buffer is shifted 
dvander@1656
   313
	 * down, and the buffer is filled from the stream reader again.
dvander@1656
   314
	 *
dvander@1656
   315
	 * What makes this particularly annoying is that we cache pointers everywhere, so when 
dvander@1656
   316
	 * the shifting process takes place, all those pointers must be shifted as well.
dvander@1656
   317
	 */
dvander@1503
   318
	while (srdr(stream, parse_point, sizeof(in_buf) - (parse_point - in_buf) - 1, &read))
dvander@591
   319
	{
dvander@591
   320
		if (!read)
dvander@591
   321
		{
dvander@591
   322
			break;
dvander@591
   323
		}
dvander@591
   324
dvander@1656
   325
		/* Check for BOM markings, which is only relevant on the first line.
dvander@1656
   326
		 * Not worth it, but it could be moved out of the loop.
dvander@591
   327
		 */
dvander@1656
   328
		if (states.line == 1 && 
dvander@591
   329
			in_buf[0] == (char)0xEF && 
dvander@591
   330
			in_buf[1] == (char)0xBB && 
dvander@591
   331
			in_buf[2] == (char)0xBF)
dvander@591
   332
		{
dvander@1143
   333
			/* Move EVERYTHING down :\ */
dvander@1143
   334
			memmove(in_buf, &in_buf[3], read - 3);
dvander@1143
   335
			read -= 3;
dvander@591
   336
		}
dvander@591
   337
dvander@591
   338
		if (reparse_point)
dvander@591
   339
		{
dvander@591
   340
			read += (parse_point - reparse_point);
dvander@591
   341
			parse_point = reparse_point;
dvander@591
   342
			reparse_point = NULL;
dvander@591
   343
		}
dvander@591
   344
dvander@591
   345
		for (i=0; i<read; i++)
dvander@591
   346
		{
dvander@591
   347
			c = parse_point[i];
dvander@591
   348
			if (c == '\n')
dvander@591
   349
			{
dvander@591
   350
				/* If we got a newline, there's a lot of things that could have happened in the interim.
dvander@591
   351
				 * First, let's make sure the staged strings are rotated.
dvander@591
   352
				 */
dvander@591
   353
				if (strings[0].ptr)
dvander@591
   354
				{
dvander@591
   355
					strings[0].end = &parse_point[i];
dvander@591
   356
					if (rotate(strings) != NULL)
dvander@591
   357
					{
dvander@1656
   358
						err = SMCError_InvalidTokens;
dvander@591
   359
						goto failed;
dvander@591
   360
					}
dvander@591
   361
				}
dvander@591
   362
dvander@591
   363
				/* Next, let's clear some line-based values that may no longer have meaning */
dvander@591
   364
				eol_comment = false;
dvander@591
   365
				in_quote = false;
dvander@591
   366
				if (ignoring && !ml_comment)
dvander@591
   367
				{
dvander@591
   368
					ignoring = false;
dvander@591
   369
				}
dvander@591
   370
dvander@1656
   371
				/* Pass the raw line onto the listener.  We terminate the line so the receiver 
dvander@1656
   372
				 * doesn't get tons of useless info.  We restore the newline after.
dvander@1656
   373
				 */
dvander@1656
   374
				parse_point[i] = '\0';
dvander@1656
   375
				if ((res=smc->ReadSMC_RawLine(&states, line_begin)) != SMCResult_Continue)
dvander@591
   376
				{
dvander@1656
   377
					err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
dvander@591
   378
					goto failed;
dvander@591
   379
				}
dvander@1656
   380
				parse_point[i] = '\n';
dvander@591
   381
dvander@591
   382
				/* Now we check the sanity of our staged strings! */
dvander@591
   383
				if (strings[2].ptr)
dvander@591
   384
				{
dvander@591
   385
					if (!curlevel)
dvander@591
   386
					{
dvander@1656
   387
						err = SMCError_InvalidProperty1;
dvander@591
   388
						goto failed;
dvander@591
   389
					}
dvander@591
   390
					/* Assume the next string is a property and pass the info on. */
dvander@591
   391
					if ((res=smc->ReadSMC_KeyValue(
dvander@1656
   392
						&states,
dvander@591
   393
						FixupString(strings[2]),
dvander@1656
   394
						FixupString(strings[1]))) != SMCResult_Continue)
dvander@591
   395
					{
dvander@1656
   396
						err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
dvander@591
   397
						goto failed;
dvander@591
   398
					}
dvander@591
   399
					scrap(strings);
dvander@591
   400
				}
dvander@591
   401
dvander@591
   402
				/* Change the states for the next line */
dvander@1656
   403
				states.col = 0;
dvander@1656
   404
				states.line++;
dvander@591
   405
				line_begin = &parse_point[i+1];		//Note: safe because this gets relocated later
dvander@1656
   406
			} 
dvander@1656
   407
			else if (ignoring) 
dvander@1656
   408
			{
dvander@591
   409
				if (in_quote)
dvander@591
   410
				{
dvander@591
   411
					/* If i was 0, this case is impossible due to reparsing */
dvander@591
   412
					if ((i != 0) && c == '"' && parse_point[i-1] != '\\')
dvander@591
   413
					{
dvander@591
   414
						/* If we reached a quote in an ignore phase,
dvander@591
   415
						 * we're staging a string and we must rotate it out.
dvander@591
   416
						 */
dvander@591
   417
						in_quote = false;
dvander@591
   418
						ignoring = false;
dvander@591
   419
						/* Set our info */
dvander@591
   420
						strings[0].end = &parse_point[i];
dvander@591
   421
						strings[0].quoted = true;
dvander@591
   422
						if (rotate(strings) != NULL)
dvander@591
   423
						{
dvander@591
   424
							/* If we rotated too many strings, there was too much crap on one line */
dvander@1656
   425
							err = SMCError_InvalidTokens;
dvander@591
   426
							goto failed;
dvander@591
   427
						}
dvander@1656
   428
					} 
dvander@1656
   429
					else if (c == '\\') 
dvander@1656
   430
					{
dvander@591
   431
						strings[0].special = true;
dvander@1073
   432
						if (i == (read - 1))
dvander@1073
   433
						{
dvander@1073
   434
							reparse_point = &parse_point[i];
dvander@1073
   435
							break;
dvander@1073
   436
						}
dvander@591
   437
					}
dvander@1656
   438
				} 
dvander@1656
   439
				else if (ml_comment) 
dvander@1656
   440
				{
dvander@591
   441
					if (c == '*')
dvander@591
   442
					{
dvander@591
   443
						/* Check if we need to get more input first */
dvander@591
   444
						if (i == read - 1)
dvander@591
   445
						{
dvander@591
   446
							reparse_point = &parse_point[i];
dvander@591
   447
							break;
dvander@591
   448
						}
dvander@591
   449
						if (parse_point[i+1] == '/')
dvander@591
   450
						{
dvander@591
   451
							ml_comment = false;
dvander@591
   452
							ignoring = false;
dvander@591
   453
							/* We should not be staging anything right now. */
dvander@591
   454
							assert(strings[0].ptr == NULL);
dvander@591
   455
							/* Advance the input stream so we don't choke on this token */
dvander@591
   456
							i++;
dvander@1656
   457
							states.col++;
dvander@591
   458
						}
dvander@591
   459
					}
dvander@591
   460
				}
dvander@1656
   461
			} 
dvander@1656
   462
			else 
dvander@1656
   463
			{
dvander@591
   464
				/* Check if we're whitespace or not */
dvander@591
   465
				if (!g_ws_chartable[(unsigned)c])
dvander@591
   466
				{
dvander@591
   467
					bool restage = false;
dvander@591
   468
					/* Check various special tokens:
dvander@591
   469
					 * ;
dvander@591
   470
					 * //
dvander@591
   471
					 * / *
dvander@591
   472
					 * {
dvander@591
   473
					 * }
dvander@591
   474
					 */
dvander@591
   475
					if (c == ';' || c == '/')
dvander@591
   476
					{
dvander@591
   477
						/* If it's a line-based comment (that is, ; or //)
dvander@591
   478
						 * we will need to scrap everything until the end of the line.
dvander@591
   479
						 */
dvander@591
   480
						if (c == '/')
dvander@591
   481
						{
dvander@591
   482
							if (i == read - 1)
dvander@591
   483
							{
dvander@591
   484
								/* If we reached the end of the look-ahead, we need to re-check our input.
dvander@591
   485
								 * Breaking out will force this to be the new reparse point!
dvander@591
   486
								 */
dvander@1109
   487
								reparse_point = &parse_point[i];
dvander@591
   488
								break;
dvander@591
   489
							}
dvander@591
   490
							if (parse_point[i + 1] == '/')
dvander@591
   491
							{
dvander@591
   492
								/* standard comment */
dvander@591
   493
								ignoring = true;
dvander@591
   494
								eol_comment = true;
dvander@591
   495
								restage = true;
dvander@1656
   496
							} 
dvander@1656
   497
							else if (parse_point[i+1] == '*') 
dvander@1656
   498
							{
dvander@591
   499
								/* inline comment - start ignoring */
dvander@591
   500
								ignoring = true;
dvander@591
   501
								ml_comment = true;
dvander@591
   502
								/* yes, we restage, meaning that:
dvander@591
   503
								 * STR/ *stuff* /ING  (space because ml comments don't nest in C++)
dvander@591
   504
								 * will not generate 'STRING', but rather 'STR' and 'ING'.
dvander@591
   505
								 * This should be a rare occurrence and is done here for convenience.
dvander@591
   506
								 */
dvander@591
   507
								restage = true;
dvander@591
   508
							}
dvander@1656
   509
						} 
dvander@1656
   510
						else 
dvander@1656
   511
						{
dvander@591
   512
							ignoring = true;
dvander@591
   513
							eol_comment = true;
dvander@591
   514
							restage = true;
dvander@591
   515
						}
dvander@1656
   516
					} 
dvander@1656
   517
					else if (c == '{') 
dvander@1656
   518
					{
dvander@591
   519
						/* If we are staging a string, we must rotate here */
dvander@591
   520
						if (strings[0].ptr)
dvander@591
   521
						{
dvander@591
   522
							/* We have unacceptable tokens on this line */
dvander@591
   523
							if (rotate(strings) != NULL)
dvander@591
   524
							{
dvander@1656
   525
								err = SMCError_InvalidSection1;
dvander@591
   526
								goto failed;
dvander@591
   527
							}
dvander@591
   528
						}
dvander@591
   529
						/* Sections must always be alone */
dvander@591
   530
						if (strings[2].ptr != NULL)
dvander@591
   531
						{
dvander@1656
   532
							err = SMCError_InvalidSection1;
dvander@591
   533
							goto failed;
dvander@1656
   534
						} 
dvander@1656
   535
						else if (strings[1].ptr == NULL)
dvander@1656
   536
						{
dvander@1656
   537
							err = SMCError_InvalidSection2;
dvander@591
   538
							goto failed;
dvander@591
   539
						}
dvander@1656
   540
						if ((res=smc->ReadSMC_NewSection(&states, FixupString(strings[1])))
dvander@1656
   541
							!= SMCResult_Continue)
dvander@591
   542
						{
dvander@1656
   543
							err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
dvander@591
   544
							goto failed;
dvander@591
   545
						}
dvander@591
   546
						strings[1] = emptystring;
dvander@591
   547
						curlevel++;
dvander@1656
   548
					} 
dvander@1656
   549
					else if (c == '}') 
dvander@1656
   550
					{
dvander@591
   551
						/* Unlike our matching friend, this can be on the same line as something prior */
dvander@591
   552
						if (rotate(strings) != NULL)
dvander@591
   553
						{
dvander@1656
   554
							err = SMCError_InvalidSection3;
dvander@591
   555
							goto failed;
dvander@591
   556
						}
dvander@591
   557
						if (strings[2].ptr)
dvander@591
   558
						{
dvander@591
   559
							if (!curlevel)
dvander@591
   560
							{
dvander@1656
   561
								err = SMCError_InvalidProperty1;
dvander@591
   562
								goto failed;
dvander@591
   563
							}
dvander@591
   564
							if ((res=smc->ReadSMC_KeyValue(
dvander@1656
   565
											&states,
dvander@591
   566
											FixupString(strings[2]),
dvander@1656
   567
											FixupString(strings[1])))
dvander@1656
   568
								!= SMCResult_Continue)
dvander@591
   569
							{
dvander@1656
   570
								err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
dvander@591
   571
								goto failed;
dvander@591
   572
							}
dvander@1656
   573
						} 
dvander@1656
   574
						else if (strings[1].ptr) 
dvander@1656
   575
						{
dvander@1656
   576
							err = SMCError_InvalidSection3;
dvander@591
   577
							goto failed;
dvander@1656
   578
						} 
dvander@1656
   579
						else if (!curlevel) 
dvander@1656
   580
						{
dvander@1656
   581
							err = SMCError_InvalidSection4;
dvander@591
   582
							goto failed;
dvander@591
   583
						}
dvander@591
   584
						/* Now it's safe to leave the section */
dvander@591
   585
						scrap(strings);
dvander@1656
   586
						if ((res=smc->ReadSMC_LeavingSection(&states)) != SMCResult_Continue)
dvander@591
   587
						{
dvander@1656
   588
							err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
dvander@591
   589
							goto failed;
dvander@591
   590
						}
dvander@591
   591
						curlevel--;
dvander@1656
   592
					} 
dvander@1656
   593
					else if (c == '"') 
dvander@1656
   594
					{
dvander@591
   595
						/* If we get a quote mark, we always restage, but we need to do it beforehand */
dvander@591
   596
						if (strings[0].ptr)
dvander@591
   597
						{
dvander@591
   598
							strings[0].end = &parse_point[i];
dvander@591
   599
							if (rotate(strings) != NULL)
dvander@591
   600
							{
dvander@1656
   601
								err = SMCError_InvalidTokens;
dvander@591
   602
								goto failed;
dvander@591
   603
							}
dvander@591
   604
						}
dvander@591
   605
						strings[0].ptr = &parse_point[i];
dvander@591
   606
						in_quote = true;
dvander@591
   607
						ignoring = true;
dvander@1656
   608
					} 
dvander@1656
   609
					else if (!strings[0].ptr) 
dvander@1656
   610
					{
dvander@591
   611
						/* If we have no string, we must start one */
dvander@591
   612
						strings[0].ptr = &parse_point[i];
dvander@591
   613
					}
dvander@591
   614
					if (restage && strings[0].ptr)
dvander@591
   615
					{
dvander@591
   616
						strings[0].end = &parse_point[i];
dvander@591
   617
						if (rotate(strings) != NULL)
dvander@591
   618
						{
dvander@1656
   619
							err = SMCError_InvalidTokens;
dvander@591
   620
							goto failed;
dvander@591
   621
						}
dvander@591
   622
					}
dvander@1656
   623
				} 
dvander@1656
   624
				else 
dvander@1656
   625
				{
dvander@591
   626
					/* If we're eating a string and get whitespace, we need to restage.
dvander@591
   627
					 * (Note that if we are quoted, this is being ignored)
dvander@591
   628
					 */
dvander@591
   629
					if (strings[0].ptr)
dvander@591
   630
					{
dvander@591
   631
						/*
dvander@591
   632
						 * The specification says the second string in a pair does not need to be quoted.
dvander@591
   633
						 * Thus, we check if there's already a string on the stack.
dvander@591
   634
						 * If there's a newline, we always rotate so the newline has an empty starter.
dvander@591
   635
						 */
dvander@591
   636
						if (!strings[1].ptr)
dvander@591
   637
						{
dvander@591
   638
							/* There's no string, so we must move this one down and eat up another */
dvander@591
   639
							strings[0].end = &parse_point[i];
dvander@591
   640
							rotate(strings);
dvander@1656
   641
						} 
dvander@1656
   642
						else if (!strings[1].quoted) 
dvander@1656
   643
						{
dvander@1656
   644
							err = SMCError_InvalidTokens;
dvander@591
   645
							goto failed;
dvander@591
   646
						}
dvander@591
   647
					}
dvander@591
   648
				}
dvander@591
   649
			}
dvander@591
   650
dvander@591
   651
			/* Advance which token we're on */
dvander@1656
   652
			states.col++;
dvander@591
   653
		}
dvander@591
   654
dvander@591
   655
		if (line_begin != in_buf)
dvander@591
   656
		{
dvander@591
   657
			/* The line buffer has advanced, so it's safe to copy N bytes back to the beginning.
dvander@591
   658
			 * What's N?  N is the lowest point we're currently relying on.
dvander@591
   659
			 */
dvander@591
   660
			char *stage = lowstring(strings);
dvander@591
   661
			if (!stage || stage > line_begin)
dvander@591
   662
			{
dvander@591
   663
				stage = line_begin;
dvander@591
   664
			}
dvander@591
   665
			unsigned int bytes = read - (stage - parse_point);
dvander@591
   666
dvander@591
   667
			/* It is now safe to delete everything before the staged point */
dvander@591
   668
			memmove(in_buf, stage, bytes);
dvander@591
   669
dvander@591
   670
			/* Calculate the number of bytes in the new buffer */
dvander@591
   671
			bytes = stage - in_buf;
dvander@591
   672
			/* Relocate all the cached pointers to our new base */
dvander@591
   673
			line_begin -= bytes;
dvander@591
   674
			reloc(strings[0], bytes);
dvander@591
   675
			reloc(strings[1], bytes);
dvander@591
   676
			reloc(strings[2], bytes);
dvander@591
   677
			if (reparse_point)
dvander@591
   678
			{
dvander@591
   679
				reparse_point -= bytes;
dvander@591
   680
			}
dvander@591
   681
			if (parse_point)
dvander@591
   682
			{
dvander@591
   683
				parse_point = &parse_point[read];
dvander@591
   684
				parse_point -= bytes;
dvander@591
   685
			}
dvander@1656
   686
		} 
dvander@1656
   687
		else if (read == sizeof(in_buf) - 1) 
dvander@1656
   688
		{
dvander@1656
   689
			err = SMCError_TokenOverflow;
dvander@591
   690
			goto failed;
dvander@591
   691
		}
dvander@591
   692
	}
dvander@591
   693
dvander@591
   694
	/* If we're done parsing and there are tokens left over... */
dvander@591
   695
	if (curlevel)
dvander@591
   696
	{
dvander@1656
   697
		err = SMCError_InvalidSection5;
dvander@591
   698
		goto failed;
dvander@1656
   699
	} 
dvander@1656
   700
	else if (strings[0].ptr || strings[1].ptr) 
dvander@1656
   701
	{
dvander@1656
   702
		err = SMCError_InvalidTokens;
dvander@591
   703
		goto failed;
dvander@591
   704
	}
dvander@591
   705
	
dvander@591
   706
	smc->ReadSMC_ParseEnd(false, false);
dvander@1656
   707
	
dvander@1656
   708
	if (pStates != NULL)
dvander@1656
   709
	{
dvander@1656
   710
		*pStates = states;
dvander@1656
   711
	}
dvander@591
   712
dvander@1656
   713
	return SMCError_Okay;
dvander@591
   714
dvander@591
   715
failed:
dvander@1656
   716
	if (pStates != NULL)
dvander@591
   717
	{
dvander@1656
   718
		*pStates = states;
dvander@591
   719
	}
dvander@591
   720
dvander@1656
   721
	smc->ReadSMC_ParseEnd(true, (err == SMCError_Custom));
dvander@591
   722
dvander@591
   723
	return err;
dvander@591
   724
}
dvander@591
   725
dvander@591
   726
dvander@591
   727
/**
dvander@591
   728
 * INI parser 
dvander@591
   729
 */
dvander@591
   730
dvander@616
   731
bool TextParsers::ParseFile_INI(const char *file, ITextListener_INI *ini_listener, unsigned int *line, unsigned int *col)
dvander@591
   732
{
dvander@591
   733
	FILE *fp = fopen(file, "rt");
dvander@591
   734
	unsigned int curline = 0;
dvander@591
   735
	unsigned int curtok;
dvander@591
   736
	size_t len;
dvander@591
   737
dvander@591
   738
	if (!fp)
dvander@591
   739
	{
dvander@591
   740
		if (line)
dvander@591
   741
		{
dvander@591
   742
			*line = 0;
dvander@591
   743
		}
dvander@591
   744
dvander@591
   745
		return false;
dvander@591
   746
	}
dvander@591
   747
dvander@591
   748
	char buffer[2048];
dvander@591
   749
	char *ptr, *save_ptr;
dvander@591
   750
	bool in_quote;
dvander@591
   751
dvander@591
   752
	while (!feof(fp))
dvander@591
   753
	{
dvander@591
   754
		curline++;
dvander@591
   755
		curtok = 0;
dvander@591
   756
		buffer[0] = '\0';
dvander@591
   757
		if (fgets(buffer, sizeof(buffer), fp) == NULL)
dvander@591
   758
		{
dvander@591
   759
			break;
dvander@591
   760
		}
dvander@591
   761
dvander@591
   762
		//:TODO: this will only run once, so find a nice way to move it out of the while loop
dvander@591
   763
		/* If this is the first line, check the first three bytes for BOM */
dvander@591
   764
		if (curline == 1 && 
dvander@591
   765
			buffer[0] == (char)0xEF && 
dvander@591
   766
			buffer[1] == (char)0xBB && 
dvander@591
   767
			buffer[2] == (char)0xBF)
dvander@591
   768
		{
dvander@591
   769
			/* We have a UTF-8 marked file... skip these bytes */
dvander@591
   770
			ptr = &buffer[3];
dvander@591
   771
		} else {
dvander@591
   772
			ptr = buffer;
dvander@591
   773
		}
dvander@591
   774
dvander@591
   775
		/***************************************************
dvander@591
   776
		 * We preprocess the string before parsing tokens! *
dvander@591
   777
		 ***************************************************/
dvander@591
   778
dvander@591
   779
		/* First strip beginning whitespace */
dvander@591
   780
		while (*ptr != '\0' && g_ws_chartable[(unsigned)*ptr] != 0)
dvander@591
   781
		{
dvander@591
   782
			ptr++;
dvander@591
   783
		}
dvander@591
   784
dvander@591
   785
		len = strlen(ptr);
dvander@591
   786
dvander@591
   787
		if (!len)
dvander@591
   788
		{
dvander@591
   789
			continue;
dvander@591
   790
		}
dvander@591
   791
dvander@591
   792
		/* Now search for comment characters */
dvander@591
   793
		in_quote = false;
dvander@591
   794
		save_ptr = ptr;
dvander@591
   795
		for (size_t i=0; i<len; i++,ptr++)
dvander@591
   796
		{
dvander@591
   797
			if (!in_quote)
dvander@591
   798
			{
dvander@591
   799
				switch (*ptr)
dvander@591
   800
				{
dvander@591
   801
				case '"':
dvander@591
   802
					{
dvander@591
   803
						in_quote = true;
dvander@591
   804
						break;
dvander@591
   805
					}
dvander@591
   806
				case ';':
dvander@591
   807
					{
dvander@591
   808
						/* Stop the loop */
dvander@591
   809
						len = i;
dvander@591
   810
						/* Terminate the string here */
dvander@591
   811
						*ptr = '\0';
dvander@591
   812
						break;
dvander@591
   813
					}
dvander@591
   814
				}
dvander@591
   815
			} else {
dvander@591
   816
				if (*ptr == '"')
dvander@591
   817
				{
dvander@591
   818
					in_quote = false;
dvander@591
   819
				}
dvander@591
   820
			}
dvander@591
   821
		}
dvander@591
   822
dvander@591
   823
		if (!len)
dvander@591
   824
		{
dvander@591
   825
			continue;
dvander@591
   826
		}
dvander@591
   827
dvander@591
   828
		ptr = save_ptr;
dvander@591
   829
dvander@591
   830
		/* Lastly, strip ending whitespace off */
dvander@591
   831
		for (size_t i=len-1; i>=0 && i<len; i--)
dvander@591
   832
		{
dvander@591
   833
			if (g_ws_chartable[(unsigned)ptr[i]])
dvander@591
   834
			{
dvander@591
   835
				ptr[i] = '\0';
dvander@591
   836
				len--;
dvander@591
   837
			} else {
dvander@591
   838
				break;
dvander@591
   839
			}
dvander@591
   840
		}
dvander@591
   841
dvander@591
   842
		if (!len)
dvander@591
   843
		{
dvander@591
   844
			continue;
dvander@591
   845
		}
dvander@591
   846
dvander@591
   847
		if (!ini_listener->ReadINI_RawLine(ptr, &curtok))
dvander@591
   848
		{
dvander@591
   849
			goto event_failed;
dvander@591
   850
		}
dvander@591
   851
dvander@591
   852
		if (*ptr == '[')
dvander@591
   853
		{
dvander@591
   854
			bool invalid_tokens = false;
dvander@591
   855
			bool got_bracket = false;
dvander@591
   856
			bool extra_tokens = false;
dvander@591
   857
			char c;
dvander@591
   858
			bool alnum;
dvander@591
   859
			wchar_t wc;
dvander@591
   860
dvander@591
   861
			for (size_t i=1; i<len; i++)
dvander@591
   862
			{
dvander@591
   863
				c = ptr[i];
dvander@591
   864
				alnum = false;
dvander@591
   865
dvander@591
   866
				if (c & (1<<7))
dvander@591
   867
				{
dvander@591
   868
					if (mbtowc(&wc, &ptr[i], len-i) != -1)
dvander@591
   869
					{
dvander@591
   870
						alnum = (iswalnum(wc) != 0);
dvander@591
   871
						i += _GetUTF8CharBytes(&ptr[i]) - 1;
dvander@591
   872
					}
dvander@591
   873
				} else {
dvander@591
   874
					alnum = (isalnum(c) != 0) || (g_ini_chartable1[(unsigned)c] != 0);
dvander@591
   875
				}
dvander@591
   876
				if (!alnum)
dvander@591
   877
				{
dvander@591
   878
					/* First check - is this a bracket? */
dvander@591
   879
					if (c == ']')
dvander@591
   880
					{
dvander@591
   881
						/* Yes! */
dvander@591
   882
						got_bracket = true;
dvander@591
   883
						/* If this isn't the last character... */
dvander@591
   884
						if (i != len - 1)
dvander@591
   885
						{
dvander@591
   886
							extra_tokens = true;
dvander@591
   887
						}
dvander@591
   888
						/* terminate */
dvander@591
   889
						ptr[i] = '\0';
dvander@591
   890
						break;
dvander@591
   891
					} else {
dvander@591
   892
						/* n...No! Continue copying. */
dvander@591
   893
						invalid_tokens = true;
dvander@591
   894
					}
dvander@591
   895
				}
dvander@591
   896
			}
dvander@591
   897
dvander@591
   898
			/* Tell the handler */
dvander@591
   899
			if (!ini_listener->ReadINI_NewSection(&ptr[1], invalid_tokens, got_bracket, extra_tokens, &curtok))
dvander@591
   900
			{
dvander@591
   901
				goto event_failed;
dvander@591
   902
			}
dvander@591
   903
		} else {
dvander@591
   904
			char *key_ptr = ptr;
dvander@591
   905
			char *val_ptr = NULL;
dvander@591
   906
			char c;
dvander@591
   907
			size_t first_space = 0;
dvander@591
   908
			bool invalid_tokens = false;
dvander@591
   909
			bool equal_token = false;
dvander@591
   910
			bool quotes = false;
dvander@591
   911
			bool alnum;
dvander@591
   912
			wchar_t wc;
dvander@591
   913
dvander@591
   914
			for (size_t i=0; i<len; i++)
dvander@591
   915
			{
dvander@591
   916
				c = ptr[i];
dvander@591
   917
				alnum = false;
dvander@591
   918
				/* is this an invalid char? */
dvander@591
   919
				if (c & (1<<7))
dvander@591
   920
				{
dvander@591
   921
					if (mbtowc(&wc, &ptr[i], len-i) != -1)
dvander@591
   922
					{
dvander@591
   923
						alnum = (iswalnum(wc) != 0);
dvander@591
   924
						i += _GetUTF8CharBytes(&ptr[i]) - 1;
dvander@591
   925
					}
dvander@591
   926
				} else {
dvander@591
   927
					alnum = (isalnum(c) != 0) || (g_ini_chartable1[(unsigned)c] != 0);
dvander@591
   928
				}
dvander@591
   929
dvander@591
   930
				if (!alnum)
dvander@591
   931
				{
dvander@591
   932
					if (g_ws_chartable[(unsigned)c])
dvander@591
   933
					{
dvander@591
   934
						/* if it's a space, keep track of the first occurring space */
dvander@591
   935
						if (!first_space)
dvander@591
   936
						{
dvander@591
   937
							first_space = i;
dvander@591
   938
						}
dvander@591
   939
					} else {
dvander@591
   940
						if (c == '=')
dvander@591
   941
						{
dvander@591
   942
							/* if it's an equal sign, we're done with the key */
dvander@591
   943
							if (first_space)
dvander@591
   944
							{
dvander@591
   945
								/* remove excess whitespace */
dvander@591
   946
								key_ptr[first_space] = '\0';
dvander@591
   947
							} else {
dvander@591
   948
								/* remove the equal sign */
dvander@591
   949
								key_ptr[i] = '\0';
dvander@591
   950
							}
dvander@591
   951
							if (ptr[++i] != '\0')
dvander@591
   952
							{
dvander@591
   953
								/* If this isn't the end, set next pointer */
dvander@591
   954
								val_ptr = &ptr[i];
dvander@591
   955
							}
dvander@591
   956
							equal_token = true;
dvander@591
   957
							break;
dvander@591
   958
						} else {
dvander@591
   959
							/* Mark that we got something invalid! */
dvander@591
   960
							invalid_tokens = true;
dvander@591
   961
							first_space = 0;
dvander@591
   962
						}
dvander@591
   963
					}
dvander@591
   964
				}
dvander@591
   965
			}
dvander@591
   966
dvander@591
   967
			/* Now we need to parse the value, if any */
dvander@591
   968
			if (val_ptr)
dvander@591
   969
			{
dvander@591
   970
				/* eat up spaces! there shouldn't be any h*/
dvander@591
   971
				while ((*val_ptr != '\0') && g_ws_chartable[(unsigned)*val_ptr] != 0)
dvander@591
   972
				{
dvander@591
   973
					val_ptr++;
dvander@591
   974
				}
dvander@591
   975
				if (*val_ptr == '\0')
dvander@591
   976
				{
dvander@591
   977
					val_ptr = NULL;
dvander@591
   978
					goto skip_value;
dvander@591
   979
				}
dvander@591
   980
				/* Do we have an initial quote? If so, the parsing rules change! */
dvander@591
   981
				if (*val_ptr == '"' && *val_ptr != '\0')
dvander@591
   982
				{
dvander@591
   983
					len = strlen(val_ptr);
dvander@591
   984
					if (val_ptr[len-1] == '"')
dvander@591
   985
					{
dvander@591
   986
						/* Strip quotes! */
dvander@591
   987
						val_ptr[--len] = '\0';
dvander@591
   988
						val_ptr++;
dvander@591
   989
						quotes = true;
dvander@591
   990
					}
dvander@591
   991
				}
dvander@591
   992
			}
dvander@591
   993
skip_value:
dvander@591
   994
			/* We're done! */
dvander@591
   995
			curtok = val_ptr - buffer;
dvander@591
   996
			if (!ini_listener->ReadINI_KeyValue(key_ptr, val_ptr, invalid_tokens, equal_token, quotes, &curtok))
dvander@591
   997
			{
dvander@591
   998
				curtok = 0;
dvander@591
   999
				goto event_failed;
dvander@591
  1000
			}
dvander@591
  1001
		}
dvander@591
  1002
	}
dvander@591
  1003
dvander@591
  1004
	if (line)
dvander@591
  1005
	{
dvander@591
  1006
		*line = curline;
dvander@591
  1007
	}
dvander@591
  1008
dvander@591
  1009
	fclose(fp);
dvander@591
  1010
dvander@591
  1011
	return true;
dvander@591
  1012
dvander@591
  1013
event_failed:
dvander@591
  1014
	if (line)
dvander@591
  1015
	{
dvander@591
  1016
		*line = curline;
dvander@591
  1017
	}
dvander@591
  1018
dvander@591
  1019
	if (col)
dvander@591
  1020
	{
dvander@591
  1021
		*col = curtok;
dvander@591
  1022
	}
dvander@591
  1023
dvander@591
  1024
	fclose(fp);
dvander@591
  1025
dvander@591
  1026
	return false;
dvander@591
  1027
}
dvander@591
  1028
dvander@1656
  1029
const char *TextParsers::GetSMCErrorString(SMCError err)
dvander@591
  1030
{
dvander@591
  1031
	static const char *s_errors[] = 
dvander@591
  1032
	{
dvander@591
  1033
		NULL,
dvander@591
  1034
		"Stream failed to open",
dvander@591
  1035
		"Stream returned read error",
dvander@591
  1036
		NULL,
dvander@591
  1037
		"Un-quoted section has invalid tokens",
dvander@591
  1038
		"Section declared without header",
dvander@591
  1039
		"Section declared with unknown tokens",
dvander@591
  1040
		"Section ending without a matching section beginning",
dvander@591
  1041
		"Section beginning without a matching ending",
dvander@591
  1042
		"Line contained too many invalid tokens",
dvander@591
  1043
		"Token buffer overflowed",
dvander@591
  1044
		"A property was declared outside of a section",
dvander@591
  1045
	};
dvander@591
  1046
dvander@1656
  1047
	if (err < SMCError_Okay || err > SMCError_InvalidProperty1)
dvander@591
  1048
	{
dvander@591
  1049
		return NULL;
dvander@591
  1050
	}
dvander@591
  1051
dvander@591
  1052
	return s_errors[err];
dvander@591
  1053
}