core/CTextParsers.cpp
author David Anderson <dvander@alliedmods.net>
Fri Dec 08 20:54:49 2006 +0000 (2006-12-08)
changeset 197 dd30d3d6a51f
parent 159 43c471413f70
child 203 c7f86f9e6f60
permissions -rw-r--r--
committed shiny new SMC parser (really Valve XML or whatever)
dvander@145
     1
#include <stdio.h>
dvander@145
     2
#include <ctype.h>
dvander@150
     3
#include <wctype.h>
dvander@145
     4
#include <string.h>
dvander@150
     5
#include <stdlib.h>
dvander@197
     6
#include <assert.h>
dvander@145
     7
#include "CTextParsers.h"
dvander@145
     8
dvander@145
     9
CTextParsers g_TextParse;
dvander@145
    10
dvander@145
    11
static int g_ini_chartable1[255] = {0};
dvander@150
    12
static int g_ws_chartable[255] = {0};
dvander@145
    13
dvander@145
    14
CTextParsers::CTextParsers()
dvander@145
    15
{
dvander@145
    16
	g_ini_chartable1['_'] = 1;
dvander@145
    17
	g_ini_chartable1['-'] = 1;
dvander@145
    18
	g_ini_chartable1[','] = 1;
dvander@145
    19
	g_ini_chartable1['+'] = 1;
dvander@145
    20
	g_ini_chartable1['.'] = 1;
dvander@145
    21
	g_ini_chartable1['$'] = 1;
dvander@145
    22
	g_ini_chartable1['?'] = 1;
dvander@145
    23
	g_ini_chartable1['/'] = 1;
dvander@150
    24
	g_ws_chartable['\n'] = 1;
dvander@150
    25
	g_ws_chartable['\v'] = 1;
dvander@150
    26
	g_ws_chartable['\r'] = 1;
dvander@150
    27
	g_ws_chartable['\t'] = 1;
dvander@150
    28
	g_ws_chartable['\f'] = 1;
dvander@150
    29
	g_ws_chartable[' '] = 1;
dvander@150
    30
}
dvander@150
    31
dvander@150
    32
unsigned int CTextParsers::GetUTF8CharBytes(const char *stream)
dvander@150
    33
{
dvander@150
    34
	return _GetUTF8CharBytes(stream);
dvander@145
    35
}
dvander@145
    36
dvander@197
    37
/**
dvander@197
    38
 * Character streams
dvander@197
    39
 */
dvander@197
    40
dvander@197
    41
struct CharStream
dvander@197
    42
{
dvander@197
    43
	const char *curpos;
dvander@197
    44
};
dvander@197
    45
dvander@197
    46
bool CharStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
dvander@197
    47
{
dvander@197
    48
	CharStream *srdr = (CharStream *)stream;
dvander@197
    49
dvander@197
    50
	const char *ptr = srdr->curpos;
dvander@197
    51
	for (size_t i=0; i<maxlength; i++)
dvander@197
    52
	{
dvander@197
    53
		if (*ptr == '\0')
dvander@197
    54
		{
dvander@197
    55
			break;
dvander@197
    56
		}
dvander@197
    57
		*buffer++ = *ptr++;
dvander@197
    58
	}
dvander@197
    59
dvander@197
    60
	*read = ptr - srdr->curpos;
dvander@197
    61
dvander@197
    62
	srdr->curpos = ptr;
dvander@197
    63
dvander@197
    64
	return true;
dvander@197
    65
}
dvander@197
    66
dvander@197
    67
SMCParseError CTextParsers::ParseString_SMC(const char *stream, 
dvander@197
    68
					 ITextListener_SMC *smc,
dvander@197
    69
					 unsigned int *line,
dvander@197
    70
					 unsigned int *col)
dvander@197
    71
{
dvander@197
    72
	CharStream srdr = { stream };
dvander@197
    73
dvander@197
    74
	return ParseStream_SMC(&srdr, CharStreamReader, smc, line, col);
dvander@197
    75
}
dvander@197
    76
dvander@197
    77
/**
dvander@197
    78
 * File streams
dvander@197
    79
 */
dvander@197
    80
dvander@197
    81
bool FileStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
dvander@197
    82
{
dvander@197
    83
	size_t num = fread(buffer, 1, maxlength, (FILE *)stream);
dvander@197
    84
dvander@197
    85
	*read = static_cast<unsigned int>(num);
dvander@197
    86
dvander@197
    87
	if (num == 0 && feof((FILE *)stream))
dvander@197
    88
	{
dvander@197
    89
		return true;
dvander@197
    90
	}
dvander@197
    91
dvander@197
    92
	return (ferror((FILE *)stream) == 0);
dvander@197
    93
}
dvander@197
    94
dvander@197
    95
SMCParseError CTextParsers::ParseFile_SMC(const char *file, ITextListener_SMC *smc, unsigned int *line, unsigned int *col)
dvander@197
    96
{
dvander@197
    97
	FILE *fp = fopen(file, "rt");
dvander@197
    98
dvander@197
    99
	if (!fp)
dvander@197
   100
	{
dvander@197
   101
		return SMCParse_StreamOpen;
dvander@197
   102
	}
dvander@197
   103
dvander@197
   104
	SMCParseError result = ParseStream_SMC(fp, FileStreamReader, smc, line, col);
dvander@197
   105
dvander@197
   106
	fclose(fp);
dvander@197
   107
dvander@197
   108
	return result;
dvander@197
   109
}
dvander@197
   110
dvander@197
   111
/** 
dvander@197
   112
 * Raw parsing of streams with helper functions
dvander@197
   113
 */
dvander@197
   114
dvander@197
   115
struct StringInfo
dvander@197
   116
{
dvander@197
   117
	StringInfo() : quoted(false), ptr(NULL), end(NULL), special(false) { }
dvander@197
   118
	char *ptr;
dvander@197
   119
	char *end;
dvander@197
   120
	bool special;
dvander@197
   121
	bool quoted;
dvander@197
   122
};
dvander@197
   123
dvander@197
   124
const char *FixupString(StringInfo &data)
dvander@197
   125
{
dvander@197
   126
	if (!data.ptr)
dvander@197
   127
	{
dvander@197
   128
		return NULL;
dvander@197
   129
	}
dvander@197
   130
dvander@197
   131
	if (data.quoted)
dvander@197
   132
	{
dvander@197
   133
		data.ptr++;
dvander@197
   134
	}
dvander@197
   135
#if defined _DEBUG
dvander@197
   136
	else {
dvander@197
   137
		/* A string will never have beginning whitespace because we ignore it in the stream.
dvander@197
   138
		 * Furthermore, if there is trailing whitespace, the end ptr will point to it, so it is valid
dvander@197
   139
		 * to overwrite!  Lastly, the last character must be whitespace or a comment/invalid character.
dvander@197
   140
		 */
dvander@197
   141
 	}
dvander@197
   142
#endif
dvander@197
   143
dvander@197
   144
	if (data.special)
dvander@197
   145
	{
dvander@197
   146
		//:TODO: this string has special tokens in it, like \, and we must
dvander@197
   147
		//resolve these before passing the string back to the app
dvander@197
   148
	}
dvander@197
   149
dvander@197
   150
	*(data.end) = '\0';
dvander@197
   151
dvander@197
   152
	return data.ptr;
dvander@197
   153
}
dvander@197
   154
dvander@197
   155
const char *rotate(StringInfo info[3])
dvander@197
   156
{
dvander@197
   157
	if (info[2].ptr != NULL)
dvander@197
   158
	{
dvander@197
   159
		return info[2].ptr;
dvander@197
   160
	}
dvander@197
   161
dvander@197
   162
	if (info[0].ptr != NULL)
dvander@197
   163
	{
dvander@197
   164
		info[2] = info[1];
dvander@197
   165
		info[1] = info[0];
dvander@197
   166
		info[0] = StringInfo();
dvander@197
   167
	}
dvander@197
   168
	
dvander@197
   169
	return NULL;
dvander@197
   170
}
dvander@197
   171
dvander@197
   172
void scrap(StringInfo info[3])
dvander@197
   173
{
dvander@197
   174
	info[2] = StringInfo();
dvander@197
   175
	info[1] = StringInfo();
dvander@197
   176
	info[0] = StringInfo();
dvander@197
   177
}
dvander@197
   178
dvander@197
   179
void reloc(StringInfo &data, unsigned int bytes)
dvander@197
   180
{
dvander@197
   181
	if (data.ptr)
dvander@197
   182
	{
dvander@197
   183
		data.ptr -= bytes;
dvander@197
   184
	}
dvander@197
   185
	if (data.end)
dvander@197
   186
	{
dvander@197
   187
		data.end -= bytes;
dvander@197
   188
	}
dvander@197
   189
}
dvander@197
   190
dvander@197
   191
char *lowstring(StringInfo info[3])
dvander@197
   192
{
dvander@197
   193
	for (int i=2; i>=0; i--)
dvander@197
   194
	{
dvander@197
   195
		if (info[i].ptr)
dvander@197
   196
		{
dvander@197
   197
			return info[i].ptr;
dvander@197
   198
		}
dvander@197
   199
	}
dvander@197
   200
dvander@197
   201
	return NULL;
dvander@197
   202
}
dvander@197
   203
dvander@197
   204
SMCParseError CTextParsers::ParseStream_SMC(void *stream, 
dvander@197
   205
								   STREAMREADER srdr, 
dvander@197
   206
								   ITextListener_SMC *smc, 
dvander@197
   207
								   unsigned int *line, 
dvander@197
   208
								   unsigned int *col)
dvander@145
   209
{
dvander@197
   210
	char in_buf[4096];
dvander@197
   211
	char *reparse_point = NULL;
dvander@197
   212
	char *parse_point = in_buf;
dvander@197
   213
	char *line_begin = in_buf;
dvander@197
   214
	unsigned int read;
dvander@197
   215
	unsigned int curline = 1;
dvander@197
   216
	unsigned int curtok = 0;
dvander@197
   217
	unsigned int curlevel = 0;
dvander@197
   218
	bool in_quote = false;
dvander@197
   219
	bool ignoring = false;
dvander@197
   220
	bool eol_comment = false;
dvander@197
   221
	bool ml_comment = false;
dvander@197
   222
	unsigned int i;
dvander@197
   223
	SMCParseError err = SMCParse_Okay;
dvander@197
   224
	SMCParseResult res;
dvander@197
   225
	char c;
dvander@197
   226
dvander@197
   227
	StringInfo strings[3];
dvander@197
   228
	StringInfo emptystring;
dvander@197
   229
dvander@197
   230
	smc->ReadSMC_ParseStart();
dvander@197
   231
dvander@197
   232
	while (srdr(stream, parse_point, sizeof(in_buf) - (parse_point - line_begin) - 1, &read))
dvander@197
   233
	{
dvander@197
   234
		if (!read)
dvander@197
   235
		{
dvander@197
   236
			err = SMCParse_StreamError;
dvander@197
   237
			goto failed;
dvander@197
   238
		}
dvander@197
   239
dvander@197
   240
		/* :TODO: do this outside of the main loop somehow
dvander@197
   241
		 * This checks for BOM markings
dvander@197
   242
		 */
dvander@197
   243
		if (curline == 1 && 
dvander@197
   244
			in_buf[0] == (char)0xEF && 
dvander@197
   245
			in_buf[1] == (char)0xBB && 
dvander@197
   246
			in_buf[2] == (char)0xBF)
dvander@197
   247
		{
dvander@197
   248
			parse_point = &in_buf[3];
dvander@197
   249
		}
dvander@197
   250
dvander@197
   251
		if (reparse_point)
dvander@197
   252
		{
dvander@197
   253
			read += (parse_point - reparse_point);
dvander@197
   254
			parse_point = reparse_point;
dvander@197
   255
			reparse_point = NULL;
dvander@197
   256
		}
dvander@197
   257
dvander@197
   258
		for (i=0; i<read; i++)
dvander@197
   259
		{
dvander@197
   260
			c = parse_point[i];
dvander@197
   261
			if (c == '\n')
dvander@197
   262
			{
dvander@197
   263
				/* If we got a newline, there's a lot of things that could have happened in the interim.
dvander@197
   264
				 * First, let's make sure the staged strings are rotated.
dvander@197
   265
				 */
dvander@197
   266
				if (strings[0].ptr)
dvander@197
   267
				{
dvander@197
   268
					strings[0].end = &parse_point[i];
dvander@197
   269
					if (rotate(strings) != NULL)
dvander@197
   270
					{
dvander@197
   271
						err = SMCParse_InvalidTokens;
dvander@197
   272
						goto failed;
dvander@197
   273
					}
dvander@197
   274
				}
dvander@197
   275
dvander@197
   276
				/* Next, let's clear some line-based values that may no longer have meaning */
dvander@197
   277
				eol_comment = false;
dvander@197
   278
				in_quote = false;
dvander@197
   279
				if (ignoring && !ml_comment)
dvander@197
   280
				{
dvander@197
   281
					ignoring = false;
dvander@197
   282
				}
dvander@197
   283
dvander@197
   284
				/* Pass the raw line onto the listener */
dvander@197
   285
				if ((res=smc->ReadSMC_RawLine(line_begin, curline)) != SMCParse_Continue)
dvander@197
   286
				{
dvander@197
   287
					err = (res == SMCParse_HaltFail) ? SMCParse_Custom : SMCParse_Okay;
dvander@197
   288
					goto failed;
dvander@197
   289
				}
dvander@197
   290
dvander@197
   291
				/* Now we check the sanity of our staged strings! */
dvander@197
   292
				if (strings[2].ptr)
dvander@197
   293
				{
dvander@197
   294
					if (!curlevel)
dvander@197
   295
					{
dvander@197
   296
						err = SMCParse_InvalidProperty1;
dvander@197
   297
						goto failed;
dvander@197
   298
					}
dvander@197
   299
					/* Assume the next string is a property and pass the info on. */
dvander@197
   300
					if ((res=smc->ReadSMC_KeyValue(
dvander@197
   301
						FixupString(strings[2]),
dvander@197
   302
						FixupString(strings[1]),
dvander@197
   303
						strings[2].quoted,
dvander@197
   304
						strings[1].quoted)) != SMCParse_Continue)
dvander@197
   305
					{
dvander@197
   306
						err = (res == SMCParse_HaltFail) ? SMCParse_Custom : SMCParse_Okay;
dvander@197
   307
						goto failed;
dvander@197
   308
					}
dvander@197
   309
					scrap(strings);
dvander@197
   310
				}
dvander@145
   311
dvander@197
   312
				/* Change the states for the next line */
dvander@197
   313
				curtok = 0;
dvander@197
   314
				curline++;
dvander@197
   315
				line_begin = &parse_point[i+1];		//Note: safe because this gets relocated later
dvander@197
   316
			} else if (ignoring) {
dvander@197
   317
				if (in_quote)
dvander@197
   318
				{
dvander@197
   319
					/* If i was 0, this case is impossible due to reparsing */
dvander@197
   320
					if ((i != 0) && c == '"' && parse_point[i-1] != '\\')
dvander@197
   321
					{
dvander@197
   322
						/* If we reached a quote in an ignore phase,
dvander@197
   323
						 * we're staging a string and we must rotate it out.
dvander@197
   324
						 */
dvander@197
   325
						in_quote = false;
dvander@197
   326
						ignoring = false;
dvander@197
   327
						/* Set our info */
dvander@197
   328
						strings[0].end = &parse_point[i];
dvander@197
   329
						strings[0].quoted = true;
dvander@197
   330
						if (rotate(strings) != NULL)
dvander@197
   331
						{
dvander@197
   332
							/* If we rotated too many strings, there was too much crap on one line */
dvander@197
   333
							err = SMCParse_InvalidTokens;
dvander@197
   334
							goto failed;
dvander@197
   335
						}
dvander@197
   336
					} else if (c == '\\' && i == (read - 1)) {
dvander@197
   337
						strings[0].special = true;
dvander@197
   338
						reparse_point = &parse_point[i];
dvander@197
   339
						break;
dvander@197
   340
					}
dvander@197
   341
				} else if (ml_comment) {
dvander@197
   342
					if (c == '*')
dvander@197
   343
					{
dvander@197
   344
						/* Check if we need to get more input first */
dvander@197
   345
						if (i == read - 1)
dvander@197
   346
						{
dvander@197
   347
							reparse_point = &parse_point[i];
dvander@197
   348
							break;
dvander@197
   349
						}
dvander@197
   350
						if (parse_point[i+1] == '/')
dvander@197
   351
						{
dvander@197
   352
							ml_comment = false;
dvander@197
   353
							ignoring = false;
dvander@197
   354
							/* We should not be staging anything right now. */
dvander@197
   355
							assert(strings[0].ptr == NULL);
dvander@197
   356
							/* Advance the input stream so we don't choke on this token */
dvander@197
   357
							i++;
dvander@197
   358
							curtok++;
dvander@197
   359
						}
dvander@197
   360
					}
dvander@197
   361
				}
dvander@197
   362
			} else {
dvander@197
   363
				/* Check if we're whitespace or not */
dvander@197
   364
				if (!g_ws_chartable[c])
dvander@197
   365
				{
dvander@197
   366
					bool restage = false;
dvander@197
   367
					/* Check various special tokens:
dvander@197
   368
					 * ;
dvander@197
   369
					 * //
dvander@197
   370
					 * /*
dvander@197
   371
					 * {
dvander@197
   372
					 * }
dvander@197
   373
					 */
dvander@197
   374
					if (c == ';' || c == '/')
dvander@197
   375
					{
dvander@197
   376
						/* If it's a line-based comment (that is, ; or //)
dvander@197
   377
						 * we will need to scrap everything until the end of the line.
dvander@197
   378
						 */
dvander@197
   379
						if (c == '/')
dvander@197
   380
						{
dvander@197
   381
							if (i == read - 1)
dvander@197
   382
							{
dvander@197
   383
								/* If we reached the end of the look-ahead, we need to re-check our input.
dvander@197
   384
								 * Breaking out will force this to be the new reparse point!
dvander@197
   385
								 */
dvander@197
   386
								reparse_point = &reparse_point[i];
dvander@197
   387
								break;
dvander@197
   388
							}
dvander@197
   389
							if (parse_point[i + 1] == '/')
dvander@197
   390
							{
dvander@197
   391
								/* standard comment */
dvander@197
   392
								ignoring = true;
dvander@197
   393
								eol_comment = true;
dvander@197
   394
								restage = true;
dvander@197
   395
							} else if (parse_point[i+1] == '*') {
dvander@197
   396
								/* inline comment - start ignoring */
dvander@197
   397
								ignoring = true;
dvander@197
   398
								ml_comment = true;
dvander@197
   399
								/* yes, we restage, meaning that:
dvander@197
   400
								 * STR/*stuff* /ING  (space because ml comments don't nest in C++)
dvander@197
   401
								 * will not generate 'STRING', but rather 'STR' and 'ING'.
dvander@197
   402
								 * This should be a rare occurrence and is done here for convenience.
dvander@197
   403
								 */
dvander@197
   404
								restage = true;
dvander@197
   405
							}
dvander@197
   406
						} else {
dvander@197
   407
							ignoring = true;
dvander@197
   408
							eol_comment = true;
dvander@197
   409
							restage = true;
dvander@197
   410
						}
dvander@197
   411
					} else if (c == '{') {
dvander@197
   412
						/* If we are staging a string, we must rotate here */
dvander@197
   413
						if (strings[0].ptr)
dvander@197
   414
						{
dvander@197
   415
							/* We have unacceptable tokens on this line */
dvander@197
   416
							if (rotate(strings) != NULL)
dvander@197
   417
							{
dvander@197
   418
								err = SMCParse_InvalidSection1;
dvander@197
   419
								goto failed;
dvander@197
   420
							}
dvander@197
   421
						}
dvander@197
   422
						/* Sections must always be alone */
dvander@197
   423
						if (strings[2].ptr != NULL)
dvander@197
   424
						{
dvander@197
   425
							err = SMCParse_InvalidSection1;
dvander@197
   426
							goto failed;
dvander@197
   427
						} else if (strings[1].ptr == NULL) {
dvander@197
   428
							err = SMCParse_InvalidSection2;
dvander@197
   429
							goto failed;
dvander@197
   430
						}
dvander@197
   431
						if ((res=smc->ReadSMC_NewSection(FixupString(strings[1]), strings[1].quoted))
dvander@197
   432
							!= SMCParse_Continue)
dvander@197
   433
						{
dvander@197
   434
							err = (res == SMCParse_HaltFail) ? SMCParse_Custom : SMCParse_Okay;
dvander@197
   435
							goto failed;
dvander@197
   436
						}
dvander@197
   437
						strings[1] = emptystring;
dvander@197
   438
						curlevel++;
dvander@197
   439
					} else if (c == '}') {
dvander@197
   440
						/* Unlike our matching friend, this can be on the same line as something prior */
dvander@197
   441
						if (rotate(strings) != NULL)
dvander@197
   442
						{
dvander@197
   443
							err = SMCParse_InvalidSection3;
dvander@197
   444
							goto failed;
dvander@197
   445
						}
dvander@197
   446
						if (strings[2].ptr)
dvander@197
   447
						{
dvander@197
   448
							if (!curlevel)
dvander@197
   449
							{
dvander@197
   450
								err = SMCParse_InvalidProperty1;
dvander@197
   451
								goto failed;
dvander@197
   452
							}
dvander@197
   453
							if ((res=smc->ReadSMC_KeyValue(
dvander@197
   454
											FixupString(strings[2]),
dvander@197
   455
											FixupString(strings[1]),
dvander@197
   456
											strings[2].quoted,
dvander@197
   457
											strings[1].quoted))
dvander@197
   458
								!= SMCParse_Continue)
dvander@197
   459
							{
dvander@197
   460
								err = (res == SMCParse_HaltFail) ? SMCParse_Custom : SMCParse_Okay;
dvander@197
   461
								goto failed;
dvander@197
   462
							}
dvander@197
   463
						} else if (strings[1].ptr) {
dvander@197
   464
							err = SMCParse_InvalidSection3;
dvander@197
   465
							goto failed;
dvander@197
   466
						} else if (!curlevel) {
dvander@197
   467
							err = SMCParse_InvalidSection4;
dvander@197
   468
							goto failed;
dvander@197
   469
						}
dvander@197
   470
						/* Now it's safe to leave the section */
dvander@197
   471
						scrap(strings);
dvander@197
   472
						if ((res=smc->ReadSMC_LeavingSection()) != SMCParse_Continue)
dvander@197
   473
						{
dvander@197
   474
							err = (res == SMCParse_HaltFail) ? SMCParse_Custom : SMCParse_Okay;
dvander@197
   475
							goto failed;
dvander@197
   476
						}
dvander@197
   477
						curlevel--;
dvander@197
   478
					} else if (c == '"') {
dvander@197
   479
						/* If we get a quote mark, we always restage, but we need to do it beforehand */
dvander@197
   480
						if (strings[0].ptr)
dvander@197
   481
						{
dvander@197
   482
							strings[0].end = &parse_point[i];
dvander@197
   483
							if (rotate(strings) != NULL)
dvander@197
   484
							{
dvander@197
   485
								err = SMCParse_InvalidTokens;
dvander@197
   486
								goto failed;
dvander@197
   487
							}
dvander@197
   488
						}
dvander@197
   489
						strings[0].ptr = &parse_point[i];
dvander@197
   490
						in_quote = true;
dvander@197
   491
						ignoring = true;
dvander@197
   492
					} else if (!strings[0].ptr) {
dvander@197
   493
						/* If we have no string, we must start one */
dvander@197
   494
						strings[0].ptr = &parse_point[i];
dvander@197
   495
					}
dvander@197
   496
					if (restage && strings[0].ptr)
dvander@197
   497
					{
dvander@197
   498
						strings[0].end = &parse_point[i];
dvander@197
   499
						if (rotate(strings) != NULL)
dvander@197
   500
						{
dvander@197
   501
							err = SMCParse_InvalidTokens;
dvander@197
   502
							goto failed;
dvander@197
   503
						}
dvander@197
   504
					}
dvander@197
   505
				} else {
dvander@197
   506
					/* If we're eating a string and get whitespace, we need to restage.
dvander@197
   507
					 * (Note that if we are quoted, this is being ignored)
dvander@197
   508
					 */
dvander@197
   509
					if (strings[0].ptr)
dvander@197
   510
					{
dvander@197
   511
						/*
dvander@197
   512
						 * The specification says the second string in a pair does not need to be quoted.
dvander@197
   513
						 * Thus, we check if there's already a string on the stack.
dvander@197
   514
						 * If there's a newline, we always rotate so the newline has an empty starter.
dvander@197
   515
						 */
dvander@197
   516
						if (!strings[1].ptr)
dvander@197
   517
						{
dvander@197
   518
							/* There's no string, so we must move this one down and eat up another */
dvander@197
   519
							strings[0].end = &parse_point[i];
dvander@197
   520
							rotate(strings);
dvander@197
   521
						} else if (!strings[1].quoted) {
dvander@197
   522
							err = SMCParse_InvalidTokens;
dvander@197
   523
							goto failed;
dvander@197
   524
						}
dvander@197
   525
					}
dvander@197
   526
				}
dvander@197
   527
			}
dvander@197
   528
dvander@197
   529
			/* Advance which token we're on */
dvander@197
   530
			curtok++;
dvander@197
   531
		}
dvander@197
   532
dvander@197
   533
		if (line_begin != in_buf)
dvander@197
   534
		{
dvander@197
   535
			/* The line buffer has advanced, so it's safe to copy N bytes back to the beginning.
dvander@197
   536
			 * What's N?  N is the lowest point we're currently relying on.
dvander@197
   537
			 */
dvander@197
   538
			char *stage = lowstring(strings);
dvander@197
   539
			if (!stage || stage > line_begin)
dvander@197
   540
			{
dvander@197
   541
				stage = line_begin;
dvander@197
   542
			}
dvander@197
   543
			unsigned int bytes = read - (stage - parse_point);
dvander@197
   544
dvander@197
   545
			/* It is now safe to delete everything before the staged point */
dvander@197
   546
			memmove(in_buf, stage, bytes);
dvander@197
   547
dvander@197
   548
			/* Calculate the number of bytes in the new buffer */
dvander@197
   549
			bytes = stage - in_buf;
dvander@197
   550
			/* Relocate all the cached pointers to our new base */
dvander@197
   551
			line_begin -= bytes;
dvander@197
   552
			reloc(strings[0], bytes);
dvander@197
   553
			reloc(strings[1], bytes);
dvander@197
   554
			reloc(strings[2], bytes);
dvander@197
   555
			if (reparse_point)
dvander@197
   556
			{
dvander@197
   557
				reparse_point -= bytes;
dvander@197
   558
			}
dvander@197
   559
			if (parse_point)
dvander@197
   560
			{
dvander@197
   561
				parse_point = &parse_point[read];
dvander@197
   562
				parse_point -= bytes;
dvander@197
   563
			}
dvander@197
   564
		} else if (read == sizeof(in_buf) - 1) {
dvander@197
   565
			err = SMCParse_TokenOverflow;
dvander@197
   566
			goto failed;
dvander@197
   567
		}
dvander@197
   568
	}
dvander@197
   569
dvander@197
   570
	/* If we're done parsing and there are tokens left over... */
dvander@197
   571
	if (curlevel)
dvander@197
   572
	{
dvander@197
   573
		err = SMCParse_InvalidSection5;
dvander@197
   574
		goto failed;
dvander@197
   575
	} else if (strings[0].ptr || strings[1].ptr) {
dvander@197
   576
		err = SMCParse_InvalidTokens;
dvander@197
   577
		goto failed;
dvander@197
   578
	}
dvander@197
   579
	
dvander@197
   580
	smc->ReadSMC_ParseEnd(false, false);
dvander@197
   581
dvander@197
   582
	return SMCParse_Okay;
dvander@197
   583
dvander@197
   584
failed:
dvander@197
   585
	if (line)
dvander@197
   586
	{
dvander@197
   587
		*line = curline;
dvander@197
   588
	}
dvander@197
   589
dvander@197
   590
	if (col)
dvander@197
   591
	{
dvander@197
   592
		*col = curtok;
dvander@197
   593
	}
dvander@197
   594
dvander@197
   595
	return err;
dvander@145
   596
}
dvander@145
   597
dvander@197
   598
dvander@197
   599
/**
dvander@197
   600
 * INI parser 
dvander@197
   601
 */
dvander@197
   602
dvander@145
   603
bool CTextParsers::ParseFile_INI(const char *file, ITextListener_INI *ini_listener, unsigned int *line, unsigned int *col)
dvander@145
   604
{
dvander@145
   605
	FILE *fp = fopen(file, "rt");
dvander@145
   606
	unsigned int curline = 0;
dvander@145
   607
	unsigned int curtok;
dvander@145
   608
	size_t len;
dvander@145
   609
dvander@145
   610
	if (!fp)
dvander@145
   611
	{
dvander@145
   612
		if (line)
dvander@145
   613
		{
dvander@145
   614
			*line = 0;
dvander@145
   615
		}
dvander@145
   616
dvander@145
   617
		return false;
dvander@145
   618
	}
dvander@145
   619
dvander@145
   620
	char buffer[2048];
dvander@145
   621
	char *ptr, *save_ptr;
dvander@145
   622
	bool in_quote;
dvander@150
   623
dvander@145
   624
	while (!feof(fp))
dvander@145
   625
	{
dvander@145
   626
		curline++;
dvander@145
   627
		curtok = 0;
dvander@145
   628
		buffer[0] = '\0';
dvander@145
   629
		if (fgets(buffer, sizeof(buffer), fp) == NULL)
dvander@145
   630
		{
dvander@145
   631
			break;
dvander@145
   632
		}
dvander@150
   633
dvander@150
   634
		//:TODO: this will only run once, so find a nice way to move it out of the while loop
dvander@150
   635
		/* If this is the first line, check the first three bytes for BOM */
dvander@150
   636
		if (curline == 1 && 
dvander@150
   637
			buffer[0] == (char)0xEF && 
dvander@150
   638
			buffer[1] == (char)0xBB && 
dvander@150
   639
			buffer[2] == (char)0xBF)
dvander@150
   640
		{
dvander@150
   641
			/* We have a UTF-8 marked file... skip these bytes */
dvander@150
   642
			ptr = &buffer[3];
dvander@150
   643
		} else {
dvander@150
   644
			ptr = buffer;
dvander@150
   645
		}
dvander@150
   646
dvander@150
   647
		/***************************************************
dvander@150
   648
		 * We preprocess the string before parsing tokens! *
dvander@150
   649
		 ***************************************************/
dvander@150
   650
dvander@145
   651
		/* First strip beginning whitespace */
dvander@150
   652
		while (*ptr != '\0' && g_ws_chartable[*ptr] != 0)
dvander@145
   653
		{
dvander@145
   654
			ptr++;
dvander@145
   655
		}
dvander@145
   656
dvander@145
   657
		len = strlen(ptr);
dvander@145
   658
dvander@145
   659
		if (!len)
dvander@145
   660
		{
dvander@145
   661
			continue;
dvander@145
   662
		}
dvander@145
   663
dvander@145
   664
		/* Now search for comment characters */
dvander@145
   665
		in_quote = false;
dvander@145
   666
		save_ptr = ptr;
dvander@145
   667
		for (size_t i=0; i<len; i++,ptr++)
dvander@145
   668
		{
dvander@145
   669
			if (!in_quote)
dvander@145
   670
			{
dvander@145
   671
				switch (*ptr)
dvander@145
   672
				{
dvander@145
   673
				case '"':
dvander@145
   674
					{
dvander@145
   675
						in_quote = true;
dvander@145
   676
						break;
dvander@145
   677
					}
dvander@145
   678
				case ';':
dvander@145
   679
					{
dvander@145
   680
						/* Stop the loop */
dvander@145
   681
						len = i;
dvander@145
   682
						/* Terminate the string here */
dvander@145
   683
						*ptr = '\0';
dvander@145
   684
						break;
dvander@145
   685
					}
dvander@145
   686
				}
dvander@145
   687
			} else {
dvander@145
   688
				if (*ptr == '"')
dvander@145
   689
				{
dvander@145
   690
					in_quote = false;
dvander@145
   691
				}
dvander@145
   692
			}
dvander@145
   693
		}
dvander@145
   694
dvander@145
   695
		if (!len)
dvander@145
   696
		{
dvander@145
   697
			continue;
dvander@145
   698
		}
dvander@145
   699
dvander@145
   700
		ptr = save_ptr;
dvander@145
   701
dvander@145
   702
		/* Lastly, strip ending whitespace off */
dvander@145
   703
		for (size_t i=len-1; i>=0 && i<len; i--)
dvander@145
   704
		{
dvander@150
   705
			if (g_ws_chartable[ptr[i]])
dvander@145
   706
			{
dvander@145
   707
				ptr[i] = '\0';
dvander@145
   708
				len--;
dvander@145
   709
			} else {
dvander@145
   710
				break;
dvander@145
   711
			}
dvander@145
   712
		}
dvander@145
   713
dvander@145
   714
		if (!len)
dvander@145
   715
		{
dvander@145
   716
			continue;
dvander@145
   717
		}
dvander@145
   718
dvander@145
   719
		if (!ini_listener->ReadINI_RawLine(ptr, &curtok))
dvander@145
   720
		{
dvander@145
   721
			goto event_failed;
dvander@145
   722
		}
dvander@145
   723
dvander@145
   724
		if (*ptr == '[')
dvander@145
   725
		{
dvander@145
   726
			bool invalid_tokens = false;
dvander@145
   727
			bool got_bracket = false;
dvander@145
   728
			bool extra_tokens = false;
dvander@145
   729
			char c;
dvander@150
   730
			bool alnum;
dvander@150
   731
			wchar_t wc;
dvander@145
   732
dvander@145
   733
			for (size_t i=1; i<len; i++)
dvander@145
   734
			{
dvander@145
   735
				c = ptr[i];
dvander@150
   736
				alnum = false;
dvander@150
   737
dvander@150
   738
				if (c & (1<<7))
dvander@150
   739
				{
dvander@150
   740
					if (mbtowc(&wc, &ptr[i], len-i) != -1)
dvander@150
   741
					{
dvander@150
   742
						alnum = (iswalnum(wc) != 0);
dvander@150
   743
						i += _GetUTF8CharBytes(&ptr[i]) - 1;
dvander@150
   744
					}
dvander@150
   745
				} else {
dvander@150
   746
					alnum = (isalnum(c) != 0) || (g_ini_chartable1[c] != 0);
dvander@150
   747
				}
dvander@150
   748
				if (!alnum)
dvander@145
   749
				{
dvander@145
   750
					/* First check - is this a bracket? */
dvander@145
   751
					if (c == ']')
dvander@145
   752
					{
dvander@145
   753
						/* Yes! */
dvander@145
   754
						got_bracket = true;
dvander@145
   755
						/* If this isn't the last character... */
dvander@145
   756
						if (i != len - 1)
dvander@145
   757
						{
dvander@145
   758
							extra_tokens = true;
dvander@145
   759
						}
dvander@145
   760
						/* terminate */
dvander@145
   761
						ptr[i] = '\0';
dvander@145
   762
						break;
dvander@145
   763
					} else {
dvander@145
   764
						/* n...No! Continue copying. */
dvander@145
   765
						invalid_tokens = true;
dvander@145
   766
					}
dvander@145
   767
				}
dvander@145
   768
			}
dvander@145
   769
dvander@145
   770
			/* Tell the handler */
dvander@145
   771
			if (!ini_listener->ReadINI_NewSection(&ptr[1], invalid_tokens, got_bracket, extra_tokens, &curtok))
dvander@145
   772
			{
dvander@145
   773
				goto event_failed;
dvander@145
   774
			}
dvander@145
   775
		} else {
dvander@145
   776
			char *key_ptr = ptr;
dvander@145
   777
			char *val_ptr = NULL;
dvander@145
   778
			char c;
dvander@145
   779
			size_t first_space = 0;
dvander@145
   780
			bool invalid_tokens = false;
dvander@145
   781
			bool equal_token = false;
dvander@145
   782
			bool quotes = false;
dvander@150
   783
			bool alnum;
dvander@150
   784
			wchar_t wc;
dvander@150
   785
dvander@145
   786
			for (size_t i=0; i<len; i++)
dvander@145
   787
			{
dvander@145
   788
				c = ptr[i];
dvander@150
   789
				alnum = false;
dvander@145
   790
				/* is this an invalid char? */
dvander@150
   791
				if (c & (1<<7))
dvander@145
   792
				{
dvander@150
   793
					if (mbtowc(&wc, &ptr[i], len-i) != -1)
dvander@150
   794
					{
dvander@150
   795
						alnum = (iswalnum(wc) != 0);
dvander@150
   796
						i += _GetUTF8CharBytes(&ptr[i]) - 1;
dvander@150
   797
					}
dvander@150
   798
				} else {
dvander@150
   799
					alnum = (isalnum(c) != 0) || (g_ini_chartable1[c] != 0);
dvander@150
   800
				}
dvander@150
   801
dvander@150
   802
				if (!alnum)
dvander@150
   803
				{
dvander@150
   804
					if (g_ws_chartable[c])
dvander@145
   805
					{
dvander@159
   806
						/* if it's a space, keep track of the first occurring space */
dvander@145
   807
						if (!first_space)
dvander@145
   808
						{
dvander@145
   809
							first_space = i;
dvander@145
   810
						}
dvander@145
   811
					} else {
dvander@145
   812
						if (c == '=')
dvander@145
   813
						{
dvander@145
   814
							/* if it's an equal sign, we're done with the key */
dvander@145
   815
							if (first_space)
dvander@145
   816
							{
dvander@145
   817
								/* remove excess whitespace */
dvander@145
   818
								key_ptr[first_space] = '\0';
dvander@145
   819
							} else {
dvander@145
   820
								/* remove the equal sign */
dvander@145
   821
								key_ptr[i] = '\0';
dvander@145
   822
							}
dvander@145
   823
							if (ptr[++i] != '\0')
dvander@145
   824
							{
dvander@145
   825
								/* If this isn't the end, set next pointer */
dvander@145
   826
								val_ptr = &ptr[i];
dvander@145
   827
							}
dvander@145
   828
							equal_token = true;
dvander@145
   829
							break;
dvander@145
   830
						} else {
dvander@145
   831
							/* Mark that we got something invalid! */
dvander@145
   832
							invalid_tokens = true;
dvander@145
   833
							first_space = 0;
dvander@145
   834
						}
dvander@145
   835
					}
dvander@145
   836
				}
dvander@145
   837
			}
dvander@145
   838
dvander@145
   839
			/* Now we need to parse the value, if any */
dvander@145
   840
			if (val_ptr)
dvander@145
   841
			{
dvander@145
   842
				/* eat up spaces! there shouldn't be any h*/
dvander@150
   843
				while ((*val_ptr != '\0') && g_ws_chartable[*val_ptr] != 0)
dvander@145
   844
				{
dvander@145
   845
					val_ptr++;
dvander@145
   846
				}
dvander@145
   847
				if (*val_ptr == '\0')
dvander@145
   848
				{
dvander@145
   849
					val_ptr = NULL;
dvander@145
   850
					goto skip_value;
dvander@145
   851
				}
dvander@145
   852
				/* Do we have an initial quote? If so, the parsing rules change! */
dvander@145
   853
				if (*val_ptr == '"' && *val_ptr != '\0')
dvander@145
   854
				{
dvander@145
   855
					len = strlen(val_ptr);
dvander@145
   856
					if (val_ptr[len-1] == '"')
dvander@145
   857
					{
dvander@145
   858
						/* Strip quotes! */
dvander@145
   859
						val_ptr[--len] = '\0';
dvander@145
   860
						val_ptr++;
dvander@145
   861
						quotes = true;
dvander@145
   862
					}
dvander@145
   863
				}
dvander@145
   864
			}
dvander@145
   865
skip_value:
dvander@145
   866
			/* We're done! */
dvander@145
   867
			curtok = val_ptr - buffer;
dvander@145
   868
			if (!ini_listener->ReadINI_KeyValue(key_ptr, val_ptr, invalid_tokens, equal_token, quotes, &curtok))
dvander@145
   869
			{
dvander@145
   870
				curtok = 0;
dvander@145
   871
				goto event_failed;
dvander@145
   872
			}
dvander@145
   873
		}
dvander@145
   874
	}
dvander@145
   875
dvander@145
   876
	if (line)
dvander@145
   877
	{
dvander@145
   878
		*line = curline;
dvander@145
   879
	}
dvander@145
   880
dvander@145
   881
	fclose(fp);
dvander@145
   882
dvander@145
   883
	return true;
dvander@145
   884
dvander@145
   885
event_failed:
dvander@145
   886
	if (line)
dvander@145
   887
	{
dvander@145
   888
		*line = curline;
dvander@145
   889
	}
dvander@145
   890
dvander@145
   891
	if (col)
dvander@145
   892
	{
dvander@145
   893
		*col = curtok;
dvander@145
   894
	}
dvander@145
   895
dvander@145
   896
	fclose(fp);
dvander@145
   897
dvander@145
   898
	return false;
dvander@145
   899
}