1.1 --- a/core/CTextParsers.cpp Mon Nov 06 22:49:12 2006 +0000
1.2 +++ b/core/CTextParsers.cpp Tue Nov 07 00:53:06 2006 +0000
1.3 @@ -1,11 +1,14 @@
1.4 #include <stdio.h>
1.5 #include <ctype.h>
1.6 +#include <wctype.h>
1.7 #include <string.h>
1.8 +#include <stdlib.h>
1.9 #include "CTextParsers.h"
1.10
1.11 CTextParsers g_TextParse;
1.12
1.13 static int g_ini_chartable1[255] = {0};
1.14 +static int g_ws_chartable[255] = {0};
1.15
1.16 CTextParsers::CTextParsers()
1.17 {
1.18 @@ -17,6 +20,17 @@
1.19 g_ini_chartable1['$'] = 1;
1.20 g_ini_chartable1['?'] = 1;
1.21 g_ini_chartable1['/'] = 1;
1.22 + g_ws_chartable['\n'] = 1;
1.23 + g_ws_chartable['\v'] = 1;
1.24 + g_ws_chartable['\r'] = 1;
1.25 + g_ws_chartable['\t'] = 1;
1.26 + g_ws_chartable['\f'] = 1;
1.27 + g_ws_chartable[' '] = 1;
1.28 +}
1.29 +
1.30 +unsigned int CTextParsers::GetUTF8CharBytes(const char *stream)
1.31 +{
1.32 + return _GetUTF8CharBytes(stream);
1.33 }
1.34
1.35 bool CTextParsers::ParseFile_SMC(const char *file, ITextListener_SMC *smc_listener, unsigned int *line, unsigned int *col)
1.36 @@ -50,6 +64,7 @@
1.37 char buffer[2048];
1.38 char *ptr, *save_ptr;
1.39 bool in_quote;
1.40 +
1.41 while (!feof(fp))
1.42 {
1.43 curline++;
1.44 @@ -59,12 +74,26 @@
1.45 {
1.46 break;
1.47 }
1.48 -
1.49 - /* Preprocess the string before anything */
1.50 - ptr = buffer;
1.51 -
1.52 +
1.53 + //:TODO: this will only run once, so find a nice way to move it out of the while loop
1.54 + /* If this is the first line, check the first three bytes for BOM */
1.55 + if (curline == 1 &&
1.56 + buffer[0] == (char)0xEF &&
1.57 + buffer[1] == (char)0xBB &&
1.58 + buffer[2] == (char)0xBF)
1.59 + {
1.60 + /* We have a UTF-8 marked file... skip these bytes */
1.61 + ptr = &buffer[3];
1.62 + } else {
1.63 + ptr = buffer;
1.64 + }
1.65 +
1.66 + /***************************************************
1.67 + * We preprocess the string before parsing tokens! *
1.68 + ***************************************************/
1.69 +
1.70 /* First strip beginning whitespace */
1.71 - while ((*ptr != '\0') && isspace(*ptr))
1.72 + while (*ptr != '\0' && g_ws_chartable[*ptr] != 0)
1.73 {
1.74 ptr++;
1.75 }
1.76 @@ -117,7 +146,7 @@
1.77 /* Lastly, strip ending whitespace off */
1.78 for (size_t i=len-1; i>=0 && i<len; i--)
1.79 {
1.80 - if (isspace(ptr[i]))
1.81 + if (g_ws_chartable[ptr[i]])
1.82 {
1.83 ptr[i] = '\0';
1.84 len--;
1.85 @@ -142,11 +171,25 @@
1.86 bool got_bracket = false;
1.87 bool extra_tokens = false;
1.88 char c;
1.89 + bool alnum;
1.90 + wchar_t wc;
1.91
1.92 for (size_t i=1; i<len; i++)
1.93 {
1.94 c = ptr[i];
1.95 - if (!isalnum(c) && !g_ini_chartable1[c])
1.96 + alnum = false;
1.97 +
1.98 + if (c & (1<<7))
1.99 + {
1.100 + if (mbtowc(&wc, &ptr[i], len-i) != -1)
1.101 + {
1.102 + alnum = (iswalnum(wc) != 0);
1.103 + i += _GetUTF8CharBytes(&ptr[i]) - 1;
1.104 + }
1.105 + } else {
1.106 + alnum = (isalnum(c) != 0) || (g_ini_chartable1[c] != 0);
1.107 + }
1.108 + if (!alnum)
1.109 {
1.110 /* First check - is this a bracket? */
1.111 if (c == ']')
1.112 @@ -181,14 +224,28 @@
1.113 bool invalid_tokens = false;
1.114 bool equal_token = false;
1.115 bool quotes = false;
1.116 -
1.117 + bool alnum;
1.118 + wchar_t wc;
1.119 +
1.120 for (size_t i=0; i<len; i++)
1.121 {
1.122 c = ptr[i];
1.123 + alnum = false;
1.124 /* is this an invalid char? */
1.125 - if (!isalnum(c) && !g_ini_chartable1[c])
1.126 + if (c & (1<<7))
1.127 {
1.128 - if (isspace(c))
1.129 + if (mbtowc(&wc, &ptr[i], len-i) != -1)
1.130 + {
1.131 + alnum = (iswalnum(wc) != 0);
1.132 + i += _GetUTF8CharBytes(&ptr[i]) - 1;
1.133 + }
1.134 + } else {
1.135 + alnum = (isalnum(c) != 0) || (g_ini_chartable1[c] != 0);
1.136 + }
1.137 +
1.138 + if (!alnum)
1.139 + {
1.140 + if (g_ws_chartable[c])
1.141 {
1.142 /* if it's a space, keep track of the last space */
1.143 if (!first_space)
1.144 @@ -227,7 +284,7 @@
1.145 if (val_ptr)
1.146 {
1.147 /* eat up spaces! there shouldn't be any h*/
1.148 - while ((*val_ptr != '\0') && isspace(*val_ptr))
1.149 + while ((*val_ptr != '\0') && g_ws_chartable[*val_ptr] != 0)
1.150 {
1.151 val_ptr++;
1.152 }
