core/CTextParsers.cpp
author David Anderson <dvander@alliedmods.net>
Tue Nov 07 00:53:06 2006 +0000 (2006-11-07)
changeset 150 42d85020e6ea
parent 145 473df5b9878e
child 159 43c471413f70
permissions -rw-r--r--
added and tested UTF-8 support for ini files
dvander@145
     1
#include <stdio.h>
dvander@145
     2
#include <ctype.h>
dvander@150
     3
#include <wctype.h>
dvander@145
     4
#include <string.h>
dvander@150
     5
#include <stdlib.h>
dvander@145
     6
#include "CTextParsers.h"
dvander@145
     7
dvander@145
     8
CTextParsers g_TextParse;
dvander@145
     9
dvander@145
    10
static int g_ini_chartable1[255] = {0};
dvander@150
    11
static int g_ws_chartable[255] = {0};
dvander@145
    12
dvander@145
    13
CTextParsers::CTextParsers()
dvander@145
    14
{
dvander@145
    15
	g_ini_chartable1['_'] = 1;
dvander@145
    16
	g_ini_chartable1['-'] = 1;
dvander@145
    17
	g_ini_chartable1[','] = 1;
dvander@145
    18
	g_ini_chartable1['+'] = 1;
dvander@145
    19
	g_ini_chartable1['.'] = 1;
dvander@145
    20
	g_ini_chartable1['$'] = 1;
dvander@145
    21
	g_ini_chartable1['?'] = 1;
dvander@145
    22
	g_ini_chartable1['/'] = 1;
dvander@150
    23
	g_ws_chartable['\n'] = 1;
dvander@150
    24
	g_ws_chartable['\v'] = 1;
dvander@150
    25
	g_ws_chartable['\r'] = 1;
dvander@150
    26
	g_ws_chartable['\t'] = 1;
dvander@150
    27
	g_ws_chartable['\f'] = 1;
dvander@150
    28
	g_ws_chartable[' '] = 1;
dvander@150
    29
}
dvander@150
    30
dvander@150
    31
unsigned int CTextParsers::GetUTF8CharBytes(const char *stream)
dvander@150
    32
{
dvander@150
    33
	return _GetUTF8CharBytes(stream);
dvander@145
    34
}
dvander@145
    35
dvander@145
    36
bool CTextParsers::ParseFile_SMC(const char *file, ITextListener_SMC *smc_listener, unsigned int *line, unsigned int *col)
dvander@145
    37
{
dvander@145
    38
	/* :TODO: Implement this */
dvander@145
    39
	if (line)
dvander@145
    40
	{
dvander@145
    41
		*line = 0;
dvander@145
    42
	}
dvander@145
    43
dvander@145
    44
	return false;
dvander@145
    45
}
dvander@145
    46
dvander@145
    47
bool CTextParsers::ParseFile_INI(const char *file, ITextListener_INI *ini_listener, unsigned int *line, unsigned int *col)
dvander@145
    48
{
dvander@145
    49
	FILE *fp = fopen(file, "rt");
dvander@145
    50
	unsigned int curline = 0;
dvander@145
    51
	unsigned int curtok;
dvander@145
    52
	size_t len;
dvander@145
    53
dvander@145
    54
	if (!fp)
dvander@145
    55
	{
dvander@145
    56
		if (line)
dvander@145
    57
		{
dvander@145
    58
			*line = 0;
dvander@145
    59
		}
dvander@145
    60
dvander@145
    61
		return false;
dvander@145
    62
	}
dvander@145
    63
dvander@145
    64
	char buffer[2048];
dvander@145
    65
	char *ptr, *save_ptr;
dvander@145
    66
	bool in_quote;
dvander@150
    67
dvander@145
    68
	while (!feof(fp))
dvander@145
    69
	{
dvander@145
    70
		curline++;
dvander@145
    71
		curtok = 0;
dvander@145
    72
		buffer[0] = '\0';
dvander@145
    73
		if (fgets(buffer, sizeof(buffer), fp) == NULL)
dvander@145
    74
		{
dvander@145
    75
			break;
dvander@145
    76
		}
dvander@150
    77
dvander@150
    78
		//:TODO: this will only run once, so find a nice way to move it out of the while loop
dvander@150
    79
		/* If this is the first line, check the first three bytes for BOM */
dvander@150
    80
		if (curline == 1 && 
dvander@150
    81
			buffer[0] == (char)0xEF && 
dvander@150
    82
			buffer[1] == (char)0xBB && 
dvander@150
    83
			buffer[2] == (char)0xBF)
dvander@150
    84
		{
dvander@150
    85
			/* We have a UTF-8 marked file... skip these bytes */
dvander@150
    86
			ptr = &buffer[3];
dvander@150
    87
		} else {
dvander@150
    88
			ptr = buffer;
dvander@150
    89
		}
dvander@150
    90
dvander@150
    91
		/***************************************************
dvander@150
    92
		 * We preprocess the string before parsing tokens! *
dvander@150
    93
		 ***************************************************/
dvander@150
    94
dvander@145
    95
		/* First strip beginning whitespace */
dvander@150
    96
		while (*ptr != '\0' && g_ws_chartable[*ptr] != 0)
dvander@145
    97
		{
dvander@145
    98
			ptr++;
dvander@145
    99
		}
dvander@145
   100
dvander@145
   101
		len = strlen(ptr);
dvander@145
   102
dvander@145
   103
		if (!len)
dvander@145
   104
		{
dvander@145
   105
			continue;
dvander@145
   106
		}
dvander@145
   107
dvander@145
   108
		/* Now search for comment characters */
dvander@145
   109
		in_quote = false;
dvander@145
   110
		save_ptr = ptr;
dvander@145
   111
		for (size_t i=0; i<len; i++,ptr++)
dvander@145
   112
		{
dvander@145
   113
			if (!in_quote)
dvander@145
   114
			{
dvander@145
   115
				switch (*ptr)
dvander@145
   116
				{
dvander@145
   117
				case '"':
dvander@145
   118
					{
dvander@145
   119
						in_quote = true;
dvander@145
   120
						break;
dvander@145
   121
					}
dvander@145
   122
				case ';':
dvander@145
   123
					{
dvander@145
   124
						/* Stop the loop */
dvander@145
   125
						len = i;
dvander@145
   126
						/* Terminate the string here */
dvander@145
   127
						*ptr = '\0';
dvander@145
   128
						break;
dvander@145
   129
					}
dvander@145
   130
				}
dvander@145
   131
			} else {
dvander@145
   132
				if (*ptr == '"')
dvander@145
   133
				{
dvander@145
   134
					in_quote = false;
dvander@145
   135
				}
dvander@145
   136
			}
dvander@145
   137
		}
dvander@145
   138
dvander@145
   139
		if (!len)
dvander@145
   140
		{
dvander@145
   141
			continue;
dvander@145
   142
		}
dvander@145
   143
dvander@145
   144
		ptr = save_ptr;
dvander@145
   145
dvander@145
   146
		/* Lastly, strip ending whitespace off */
dvander@145
   147
		for (size_t i=len-1; i>=0 && i<len; i--)
dvander@145
   148
		{
dvander@150
   149
			if (g_ws_chartable[ptr[i]])
dvander@145
   150
			{
dvander@145
   151
				ptr[i] = '\0';
dvander@145
   152
				len--;
dvander@145
   153
			} else {
dvander@145
   154
				break;
dvander@145
   155
			}
dvander@145
   156
		}
dvander@145
   157
dvander@145
   158
		if (!len)
dvander@145
   159
		{
dvander@145
   160
			continue;
dvander@145
   161
		}
dvander@145
   162
dvander@145
   163
		if (!ini_listener->ReadINI_RawLine(ptr, &curtok))
dvander@145
   164
		{
dvander@145
   165
			goto event_failed;
dvander@145
   166
		}
dvander@145
   167
dvander@145
   168
		if (*ptr == '[')
dvander@145
   169
		{
dvander@145
   170
			bool invalid_tokens = false;
dvander@145
   171
			bool got_bracket = false;
dvander@145
   172
			bool extra_tokens = false;
dvander@145
   173
			char c;
dvander@150
   174
			bool alnum;
dvander@150
   175
			wchar_t wc;
dvander@145
   176
dvander@145
   177
			for (size_t i=1; i<len; i++)
dvander@145
   178
			{
dvander@145
   179
				c = ptr[i];
dvander@150
   180
				alnum = false;
dvander@150
   181
dvander@150
   182
				if (c & (1<<7))
dvander@150
   183
				{
dvander@150
   184
					if (mbtowc(&wc, &ptr[i], len-i) != -1)
dvander@150
   185
					{
dvander@150
   186
						alnum = (iswalnum(wc) != 0);
dvander@150
   187
						i += _GetUTF8CharBytes(&ptr[i]) - 1;
dvander@150
   188
					}
dvander@150
   189
				} else {
dvander@150
   190
					alnum = (isalnum(c) != 0) || (g_ini_chartable1[c] != 0);
dvander@150
   191
				}
dvander@150
   192
				if (!alnum)
dvander@145
   193
				{
dvander@145
   194
					/* First check - is this a bracket? */
dvander@145
   195
					if (c == ']')
dvander@145
   196
					{
dvander@145
   197
						/* Yes! */
dvander@145
   198
						got_bracket = true;
dvander@145
   199
						/* If this isn't the last character... */
dvander@145
   200
						if (i != len - 1)
dvander@145
   201
						{
dvander@145
   202
							extra_tokens = true;
dvander@145
   203
						}
dvander@145
   204
						/* terminate */
dvander@145
   205
						ptr[i] = '\0';
dvander@145
   206
						break;
dvander@145
   207
					} else {
dvander@145
   208
						/* n...No! Continue copying. */
dvander@145
   209
						invalid_tokens = true;
dvander@145
   210
					}
dvander@145
   211
				}
dvander@145
   212
			}
dvander@145
   213
dvander@145
   214
			/* Tell the handler */
dvander@145
   215
			if (!ini_listener->ReadINI_NewSection(&ptr[1], invalid_tokens, got_bracket, extra_tokens, &curtok))
dvander@145
   216
			{
dvander@145
   217
				goto event_failed;
dvander@145
   218
			}
dvander@145
   219
		} else {
dvander@145
   220
			char *key_ptr = ptr;
dvander@145
   221
			char *val_ptr = NULL;
dvander@145
   222
			char c;
dvander@145
   223
			size_t first_space = 0;
dvander@145
   224
			bool invalid_tokens = false;
dvander@145
   225
			bool equal_token = false;
dvander@145
   226
			bool quotes = false;
dvander@150
   227
			bool alnum;
dvander@150
   228
			wchar_t wc;
dvander@150
   229
dvander@145
   230
			for (size_t i=0; i<len; i++)
dvander@145
   231
			{
dvander@145
   232
				c = ptr[i];
dvander@150
   233
				alnum = false;
dvander@145
   234
				/* is this an invalid char? */
dvander@150
   235
				if (c & (1<<7))
dvander@145
   236
				{
dvander@150
   237
					if (mbtowc(&wc, &ptr[i], len-i) != -1)
dvander@150
   238
					{
dvander@150
   239
						alnum = (iswalnum(wc) != 0);
dvander@150
   240
						i += _GetUTF8CharBytes(&ptr[i]) - 1;
dvander@150
   241
					}
dvander@150
   242
				} else {
dvander@150
   243
					alnum = (isalnum(c) != 0) || (g_ini_chartable1[c] != 0);
dvander@150
   244
				}
dvander@150
   245
dvander@150
   246
				if (!alnum)
dvander@150
   247
				{
dvander@150
   248
					if (g_ws_chartable[c])
dvander@145
   249
					{
dvander@145
   250
						/* if it's a space, keep track of the last space */
dvander@145
   251
						if (!first_space)
dvander@145
   252
						{
dvander@145
   253
							first_space = i;
dvander@145
   254
						}
dvander@145
   255
					} else {
dvander@145
   256
						if (c == '=')
dvander@145
   257
						{
dvander@145
   258
							/* if it's an equal sign, we're done with the key */
dvander@145
   259
							if (first_space)
dvander@145
   260
							{
dvander@145
   261
								/* remove excess whitespace */
dvander@145
   262
								key_ptr[first_space] = '\0';
dvander@145
   263
							} else {
dvander@145
   264
								/* remove the equal sign */
dvander@145
   265
								key_ptr[i] = '\0';
dvander@145
   266
							}
dvander@145
   267
							if (ptr[++i] != '\0')
dvander@145
   268
							{
dvander@145
   269
								/* If this isn't the end, set next pointer */
dvander@145
   270
								val_ptr = &ptr[i];
dvander@145
   271
							}
dvander@145
   272
							equal_token = true;
dvander@145
   273
							break;
dvander@145
   274
						} else {
dvander@145
   275
							/* Mark that we got something invalid! */
dvander@145
   276
							invalid_tokens = true;
dvander@145
   277
							first_space = 0;
dvander@145
   278
						}
dvander@145
   279
					}
dvander@145
   280
				}
dvander@145
   281
			}
dvander@145
   282
dvander@145
   283
			/* Now we need to parse the value, if any */
dvander@145
   284
			if (val_ptr)
dvander@145
   285
			{
dvander@145
   286
				/* eat up spaces! there shouldn't be any h*/
dvander@150
   287
				while ((*val_ptr != '\0') && g_ws_chartable[*val_ptr] != 0)
dvander@145
   288
				{
dvander@145
   289
					val_ptr++;
dvander@145
   290
				}
dvander@145
   291
				if (*val_ptr == '\0')
dvander@145
   292
				{
dvander@145
   293
					val_ptr = NULL;
dvander@145
   294
					goto skip_value;
dvander@145
   295
				}
dvander@145
   296
				/* Do we have an initial quote? If so, the parsing rules change! */
dvander@145
   297
				if (*val_ptr == '"' && *val_ptr != '\0')
dvander@145
   298
				{
dvander@145
   299
					len = strlen(val_ptr);
dvander@145
   300
					if (val_ptr[len-1] == '"')
dvander@145
   301
					{
dvander@145
   302
						/* Strip quotes! */
dvander@145
   303
						val_ptr[--len] = '\0';
dvander@145
   304
						val_ptr++;
dvander@145
   305
						quotes = true;
dvander@145
   306
					}
dvander@145
   307
				}
dvander@145
   308
			}
dvander@145
   309
skip_value:
dvander@145
   310
			/* We're done! */
dvander@145
   311
			curtok = val_ptr - buffer;
dvander@145
   312
			if (!ini_listener->ReadINI_KeyValue(key_ptr, val_ptr, invalid_tokens, equal_token, quotes, &curtok))
dvander@145
   313
			{
dvander@145
   314
				curtok = 0;
dvander@145
   315
				goto event_failed;
dvander@145
   316
			}
dvander@145
   317
		}
dvander@145
   318
	}
dvander@145
   319
dvander@145
   320
	if (line)
dvander@145
   321
	{
dvander@145
   322
		*line = curline;
dvander@145
   323
	}
dvander@145
   324
dvander@145
   325
	fclose(fp);
dvander@145
   326
dvander@145
   327
	return true;
dvander@145
   328
dvander@145
   329
event_failed:
dvander@145
   330
	if (line)
dvander@145
   331
	{
dvander@145
   332
		*line = curline;
dvander@145
   333
	}
dvander@145
   334
dvander@145
   335
	if (col)
dvander@145
   336
	{
dvander@145
   337
		*col = curtok;
dvander@145
   338
	}
dvander@145
   339
dvander@145
   340
	fclose(fp);
dvander@145
   341
dvander@145
   342
	return false;
dvander@145
   343
}