fixed a rare parsing bug where too many tokens would be detected if the staging buffer was broken directly before the second quotation mark in a key string
3 * =============================================================================
5 * Copyright (C) 2004-2007 AlliedModders LLC. All rights reserved.
6 * =============================================================================
8 * This program is free software; you can redistribute it and/or modify it under
9 * the terms of the GNU General Public License, version 3.0, as published by the
10 * Free Software Foundation.
12 * This program is distributed in the hope that it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
17 * You should have received a copy of the GNU General Public License along with
18 * this program. If not, see <http://www.gnu.org/licenses/>.
20 * As a special exception, AlliedModders LLC gives you permission to link the
21 * code of this program (as well as its derivative works) to "Half-Life 2," the
22 * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software
23 * by the Valve Corporation. You must obey the GNU General Public License in
24 * all respects for all other code used. Additionally, AlliedModders LLC grants
25 * this exception to all derivative works. AlliedModders LLC defines further
26 * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),
27 * or <http://www.sourcemod.net/license.php>.
38 #include "TextParsers.h"
41 TextParsers g_TextParser;
42 ITextParsers *textparsers = &g_TextParser;
44 static int g_ini_chartable1[255] = {0};
45 static int g_ws_chartable[255] = {0};
47 bool TextParsers::IsWhitespace(const char *stream)
49 return g_ws_chartable[(unsigned)*stream] == 1;
52 TextParsers::TextParsers()
54 g_ini_chartable1[(unsigned)'_'] = 1;
55 g_ini_chartable1[(unsigned)'-'] = 1;
56 g_ini_chartable1[(unsigned)','] = 1;
57 g_ini_chartable1[(unsigned)'+'] = 1;
58 g_ini_chartable1[(unsigned)'.'] = 1;
59 g_ini_chartable1[(unsigned)'$'] = 1;
60 g_ini_chartable1[(unsigned)'?'] = 1;
61 g_ini_chartable1[(unsigned)'/'] = 1;
62 g_ws_chartable[(unsigned)'\n'] = 1;
63 g_ws_chartable[(unsigned)'\v'] = 1;
64 g_ws_chartable[(unsigned)'\r'] = 1;
65 g_ws_chartable[(unsigned)'\t'] = 1;
66 g_ws_chartable[(unsigned)'\f'] = 1;
67 g_ws_chartable[(unsigned)' '] = 1;
70 void TextParsers::OnSourceModAllInitialized()
72 g_ShareSys.AddInterface(NULL, this);
75 unsigned int TextParsers::GetUTF8CharBytes(const char *stream)
77 return _GetUTF8CharBytes(stream);
89 bool CharStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
91 CharStream *srdr = (CharStream *)stream;
93 const char *ptr = srdr->curpos;
94 for (size_t i=0; i<maxlength; i++)
103 *read = ptr - srdr->curpos;
110 SMCError TextParsers::ParseString_SMC(const char *stream,
111 ITextListener_SMC *smc,
114 CharStream srdr = { stream };
116 return ParseStream_SMC(&srdr, CharStreamReader, smc, states);
123 bool FileStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
125 size_t num = fread(buffer, 1, maxlength, (FILE *)stream);
127 *read = static_cast<unsigned int>(num);
129 if (num == 0 && feof((FILE *)stream))
134 return (ferror((FILE *)stream) == 0);
137 SMCError TextParsers::ParseFile_SMC(const char *file, ITextListener_SMC *smc, SMCStates *states)
139 FILE *fp = fopen(file, "rt");
148 return SMCError_StreamOpen;
151 SMCError result = ParseStream_SMC(fp, FileStreamReader, smc, states);
159 * Raw parsing of streams with helper functions
164 StringInfo() : quoted(false), ptr(NULL), end(NULL), special(false) { }
171 const char *FixupString(StringInfo &data)
184 /* A string will never have beginning whitespace because we ignore it in the stream.
185 * Furthermore, if there is trailing whitespace, the end ptr will point to it, so it is valid
186 * to overwrite! Lastly, the last character must be whitespace or a comment/invalid character.
191 /* Do some extra work on strings that have special quoted characters. */
194 char *outptr = data.ptr;
195 size_t len = data.end - data.ptr;
198 for (size_t i=0; i<len; i++)
200 if (data.ptr[i] == '\\' && i < len - 1)
202 /* Resolve the next character. */
204 if (data.ptr[i] == 'n')
207 } else if (data.ptr[i] == 't') {
209 } else if (data.ptr[i] == 'r') {
211 } else if (data.ptr[i] != '\\'
212 && data.ptr[i] != '"') {
213 /* This character is invalid, so go back one */
217 *outptr++ = data.ptr[i];
231 const char *rotate(StringInfo info[3])
233 if (info[2].ptr != NULL)
238 if (info[0].ptr != NULL)
242 info[0] = StringInfo();
248 void scrap(StringInfo info[3])
250 info[2] = StringInfo();
251 info[1] = StringInfo();
252 info[0] = StringInfo();
255 void reloc(StringInfo &data, unsigned int bytes)
267 char *lowstring(StringInfo info[3])
269 for (int i=2; i>=0; i--)
280 SMCError TextParsers::ParseStream_SMC(void *stream,
282 ITextListener_SMC *smc,
285 char *reparse_point = NULL;
287 char *parse_point = in_buf;
288 char *line_begin = in_buf;
290 unsigned int curlevel = 0;
291 bool in_quote = false;
292 bool ignoring = false;
293 bool eol_comment = false;
294 bool ml_comment = false;
296 SMCError err = SMCError_Okay;
301 StringInfo strings[3];
302 StringInfo emptystring;
307 smc->ReadSMC_ParseStart();
310 * The stream reader reads in as much as it can fill the buffer with.
311 * It then processes the buffer. If the buffer cannot be fully processed, for example,
312 * a line is left hanging with no newline, then the contents of the buffer is shifted
313 * down, and the buffer is filled from the stream reader again.
315 * What makes this particularly annoying is that we cache pointers everywhere, so when
316 * the shifting process takes place, all those pointers must be shifted as well.
318 while (srdr(stream, parse_point, sizeof(in_buf) - (parse_point - in_buf) - 1, &read))
325 /* Check for BOM markings, which is only relevant on the first line.
326 * Not worth it, but it could be moved out of the loop.
328 if (states.line == 1 &&
329 in_buf[0] == (char)0xEF &&
330 in_buf[1] == (char)0xBB &&
331 in_buf[2] == (char)0xBF)
333 /* Move EVERYTHING down :\ */
334 memmove(in_buf, &in_buf[3], read - 3);
340 read += (parse_point - reparse_point);
341 parse_point = reparse_point;
342 reparse_point = NULL;
345 for (i=0; i<read; i++)
350 /* If we got a newline, there's a lot of things that could have happened in the interim.
351 * First, let's make sure the staged strings are rotated.
355 strings[0].end = &parse_point[i];
356 if (rotate(strings) != NULL)
358 err = SMCError_InvalidTokens;
363 /* Next, let's clear some line-based values that may no longer have meaning */
366 if (ignoring && !ml_comment)
371 /* Pass the raw line onto the listener. We terminate the line so the receiver
372 * doesn't get tons of useless info. We restore the newline after.
374 parse_point[i] = '\0';
375 if ((res=smc->ReadSMC_RawLine(&states, line_begin)) != SMCResult_Continue)
377 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
380 parse_point[i] = '\n';
382 /* Now we check the sanity of our staged strings! */
387 err = SMCError_InvalidProperty1;
390 /* Assume the next string is a property and pass the info on. */
391 if ((res=smc->ReadSMC_KeyValue(
393 FixupString(strings[2]),
394 FixupString(strings[1]))) != SMCResult_Continue)
396 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
402 /* Change the states for the next line */
405 line_begin = &parse_point[i+1]; //Note: safe because this gets relocated later
411 /* If i was 0, we could have reparsed, so make sure there's no buffer underrun */
412 if ((&parse_point[i] != in_buf) && c == '"' && parse_point[i-1] != '\\')
414 /* If we reached a quote in an ignore phase,
415 * we're staging a string and we must rotate it out.
420 strings[0].end = &parse_point[i];
421 strings[0].quoted = true;
422 if (rotate(strings) != NULL)
424 /* If we rotated too many strings, there was too much crap on one line */
425 err = SMCError_InvalidTokens;
431 strings[0].special = true;
434 reparse_point = &parse_point[i];
443 /* Check if we need to get more input first */
446 reparse_point = &parse_point[i];
449 if (parse_point[i+1] == '/')
453 /* We should not be staging anything right now. */
454 assert(strings[0].ptr == NULL);
455 /* Advance the input stream so we don't choke on this token */
464 /* Check if we're whitespace or not */
465 if (!g_ws_chartable[(unsigned)c])
467 bool restage = false;
468 /* Check various special tokens:
475 if (c == ';' || c == '/')
477 /* If it's a line-based comment (that is, ; or //)
478 * we will need to scrap everything until the end of the line.
484 /* If we reached the end of the look-ahead, we need to re-check our input.
485 * Breaking out will force this to be the new reparse point!
487 reparse_point = &parse_point[i];
490 if (parse_point[i + 1] == '/')
492 /* standard comment */
497 else if (parse_point[i+1] == '*')
499 /* inline comment - start ignoring */
502 /* yes, we restage, meaning that:
503 * STR/ *stuff* /ING (space because ml comments don't nest in C++)
504 * will not generate 'STRING', but rather 'STR' and 'ING'.
505 * This should be a rare occurrence and is done here for convenience.
519 /* If we are staging a string, we must rotate here */
522 /* We have unacceptable tokens on this line */
523 if (rotate(strings) != NULL)
525 err = SMCError_InvalidSection1;
529 /* Sections must always be alone */
530 if (strings[2].ptr != NULL)
532 err = SMCError_InvalidSection1;
535 else if (strings[1].ptr == NULL)
537 err = SMCError_InvalidSection2;
540 if ((res=smc->ReadSMC_NewSection(&states, FixupString(strings[1])))
541 != SMCResult_Continue)
543 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
546 strings[1] = emptystring;
551 /* Unlike our matching friend, this can be on the same line as something prior */
552 if (rotate(strings) != NULL)
554 err = SMCError_InvalidSection3;
561 err = SMCError_InvalidProperty1;
564 if ((res=smc->ReadSMC_KeyValue(
566 FixupString(strings[2]),
567 FixupString(strings[1])))
568 != SMCResult_Continue)
570 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
574 else if (strings[1].ptr)
576 err = SMCError_InvalidSection3;
581 err = SMCError_InvalidSection4;
584 /* Now it's safe to leave the section */
586 if ((res=smc->ReadSMC_LeavingSection(&states)) != SMCResult_Continue)
588 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
595 /* If we get a quote mark, we always restage, but we need to do it beforehand */
598 strings[0].end = &parse_point[i];
599 if (rotate(strings) != NULL)
601 err = SMCError_InvalidTokens;
605 strings[0].ptr = &parse_point[i];
609 else if (!strings[0].ptr)
611 /* If we have no string, we must start one */
612 strings[0].ptr = &parse_point[i];
614 if (restage && strings[0].ptr)
616 strings[0].end = &parse_point[i];
617 if (rotate(strings) != NULL)
619 err = SMCError_InvalidTokens;
626 /* If we're eating a string and get whitespace, we need to restage.
627 * (Note that if we are quoted, this is being ignored)
632 * The specification says the second string in a pair does not need to be quoted.
633 * Thus, we check if there's already a string on the stack.
634 * If there's a newline, we always rotate so the newline has an empty starter.
638 /* There's no string, so we must move this one down and eat up another */
639 strings[0].end = &parse_point[i];
642 else if (!strings[1].quoted)
644 err = SMCError_InvalidTokens;
651 /* Advance which token we're on */
655 if (line_begin != in_buf)
657 /* The line buffer has advanced, so it's safe to copy N bytes back to the beginning.
658 * What's N? N is the lowest point we're currently relying on.
660 char *stage = lowstring(strings);
661 if (!stage || stage > line_begin)
665 unsigned int bytes = read - (stage - parse_point);
667 /* It is now safe to delete everything before the staged point */
668 memmove(in_buf, stage, bytes);
670 /* Calculate the number of bytes in the new buffer */
671 bytes = stage - in_buf;
672 /* Relocate all the cached pointers to our new base */
674 reloc(strings[0], bytes);
675 reloc(strings[1], bytes);
676 reloc(strings[2], bytes);
679 reparse_point -= bytes;
683 parse_point = &parse_point[read];
684 parse_point -= bytes;
687 else if (read == sizeof(in_buf) - 1)
689 err = SMCError_TokenOverflow;
694 /* If we're done parsing and there are tokens left over... */
697 err = SMCError_InvalidSection5;
700 else if (strings[0].ptr || strings[1].ptr)
702 err = SMCError_InvalidTokens;
706 smc->ReadSMC_ParseEnd(false, false);
713 return SMCError_Okay;
721 smc->ReadSMC_ParseEnd(true, (err == SMCError_Custom));
731 bool TextParsers::ParseFile_INI(const char *file, ITextListener_INI *ini_listener, unsigned int *line, unsigned int *col)
733 FILE *fp = fopen(file, "rt");
734 unsigned int curline = 0;
749 char *ptr, *save_ptr;
757 if (fgets(buffer, sizeof(buffer), fp) == NULL)
762 //:TODO: this will only run once, so find a nice way to move it out of the while loop
763 /* If this is the first line, check the first three bytes for BOM */
765 buffer[0] == (char)0xEF &&
766 buffer[1] == (char)0xBB &&
767 buffer[2] == (char)0xBF)
769 /* We have a UTF-8 marked file... skip these bytes */
775 /***************************************************
776 * We preprocess the string before parsing tokens! *
777 ***************************************************/
779 /* First strip beginning whitespace */
780 while (*ptr != '\0' && g_ws_chartable[(unsigned)*ptr] != 0)
792 /* Now search for comment characters */
795 for (size_t i=0; i<len; i++,ptr++)
810 /* Terminate the string here */
830 /* Lastly, strip ending whitespace off */
831 for (size_t i=len-1; i>=0 && i<len; i--)
833 if (g_ws_chartable[(unsigned)ptr[i]])
847 if (!ini_listener->ReadINI_RawLine(ptr, &curtok))
854 bool invalid_tokens = false;
855 bool got_bracket = false;
856 bool extra_tokens = false;
861 for (size_t i=1; i<len; i++)
868 if (mbtowc(&wc, &ptr[i], len-i) != -1)
870 alnum = (iswalnum(wc) != 0);
871 i += _GetUTF8CharBytes(&ptr[i]) - 1;
874 alnum = (isalnum(c) != 0) || (g_ini_chartable1[(unsigned)c] != 0);
878 /* First check - is this a bracket? */
883 /* If this isn't the last character... */
892 /* n...No! Continue copying. */
893 invalid_tokens = true;
898 /* Tell the handler */
899 if (!ini_listener->ReadINI_NewSection(&ptr[1], invalid_tokens, got_bracket, extra_tokens, &curtok))
905 char *val_ptr = NULL;
907 size_t first_space = 0;
908 bool invalid_tokens = false;
909 bool equal_token = false;
914 for (size_t i=0; i<len; i++)
918 /* is this an invalid char? */
921 if (mbtowc(&wc, &ptr[i], len-i) != -1)
923 alnum = (iswalnum(wc) != 0);
924 i += _GetUTF8CharBytes(&ptr[i]) - 1;
927 alnum = (isalnum(c) != 0) || (g_ini_chartable1[(unsigned)c] != 0);
932 if (g_ws_chartable[(unsigned)c])
934 /* if it's a space, keep track of the first occurring space */
942 /* if it's an equal sign, we're done with the key */
945 /* remove excess whitespace */
946 key_ptr[first_space] = '\0';
948 /* remove the equal sign */
951 if (ptr[++i] != '\0')
953 /* If this isn't the end, set next pointer */
959 /* Mark that we got something invalid! */
960 invalid_tokens = true;
967 /* Now we need to parse the value, if any */
970 /* eat up spaces! there shouldn't be any h*/
971 while ((*val_ptr != '\0') && g_ws_chartable[(unsigned)*val_ptr] != 0)
975 if (*val_ptr == '\0')
980 /* Do we have an initial quote? If so, the parsing rules change! */
981 if (*val_ptr == '"' && *val_ptr != '\0')
983 len = strlen(val_ptr);
984 if (val_ptr[len-1] == '"')
987 val_ptr[--len] = '\0';
995 curtok = val_ptr - buffer;
996 if (!ini_listener->ReadINI_KeyValue(key_ptr, val_ptr, invalid_tokens, equal_token, quotes, &curtok))
1029 const char *TextParsers::GetSMCErrorString(SMCError err)
1031 static const char *s_errors[] =
1034 "Stream failed to open",
1035 "Stream returned read error",
1037 "Un-quoted section has invalid tokens",
1038 "Section declared without header",
1039 "Section declared with unknown tokens",
1040 "Section ending without a matching section beginning",
1041 "Section beginning without a matching ending",
1042 "Line contained too many invalid tokens",
1043 "Token buffer overflowed",
1044 "A property was declared outside of a section",
1047 if (err < SMCError_Okay || err > SMCError_InvalidProperty1)
1052 return s_errors[err];