3 * =============================================================================
5 * Copyright (C) 2004-2008 AlliedModders LLC. All rights reserved.
6 * =============================================================================
8 * This program is free software; you can redistribute it and/or modify it under
9 * the terms of the GNU General Public License, version 3.0, as published by the
10 * Free Software Foundation.
12 * This program is distributed in the hope that it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
17 * You should have received a copy of the GNU General Public License along with
18 * this program. If not, see <http://www.gnu.org/licenses/>.
20 * As a special exception, AlliedModders LLC gives you permission to link the
21 * code of this program (as well as its derivative works) to "Half-Life 2," the
22 * "Source Engine," the "SourcePawn JIT," and any Game MODs that run on software
23 * by the Valve Corporation. You must obey the GNU General Public License in
24 * all respects for all other code used. Additionally, AlliedModders LLC grants
25 * this exception to all derivative works. AlliedModders LLC defines further
26 * exceptions, found in LICENSE.txt (as of this writing, version JULY-31-2007),
27 * or <http://www.sourcemod.net/license.php>.
38 #include "TextParsers.h"
40 #include "sm_stringutil.h"
41 #include "LibrarySys.h"
43 TextParsers g_TextParser;
44 ITextParsers *textparsers = &g_TextParser;
46 static int g_ini_chartable1[255] = {0};
47 static int g_ws_chartable[255] = {0};
49 bool TextParsers::IsWhitespace(const char *stream)
51 return g_ws_chartable[(unsigned)*stream] == 1;
54 TextParsers::TextParsers()
56 g_ini_chartable1[(unsigned)'_'] = 1;
57 g_ini_chartable1[(unsigned)'-'] = 1;
58 g_ini_chartable1[(unsigned)','] = 1;
59 g_ini_chartable1[(unsigned)'+'] = 1;
60 g_ini_chartable1[(unsigned)'.'] = 1;
61 g_ini_chartable1[(unsigned)'$'] = 1;
62 g_ini_chartable1[(unsigned)'?'] = 1;
63 g_ini_chartable1[(unsigned)'/'] = 1;
64 g_ws_chartable[(unsigned)'\n'] = 1;
65 g_ws_chartable[(unsigned)'\v'] = 1;
66 g_ws_chartable[(unsigned)'\r'] = 1;
67 g_ws_chartable[(unsigned)'\t'] = 1;
68 g_ws_chartable[(unsigned)'\f'] = 1;
69 g_ws_chartable[(unsigned)' '] = 1;
72 void TextParsers::OnSourceModAllInitialized()
74 g_ShareSys.AddInterface(NULL, this);
77 unsigned int TextParsers::GetUTF8CharBytes(const char *stream)
79 return _GetUTF8CharBytes(stream);
91 bool CharStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
93 CharStream *srdr = (CharStream *)stream;
95 const char *ptr = srdr->curpos;
96 for (size_t i=0; i<maxlength; i++)
105 *read = ptr - srdr->curpos;
112 SMCError TextParsers::ParseString_SMC(const char *stream,
113 ITextListener_SMC *smc,
116 CharStream srdr = { stream };
118 return ParseStream_SMC(&srdr, CharStreamReader, smc, states);
125 bool FileStreamReader(void *stream, char *buffer, size_t maxlength, unsigned int *read)
127 size_t num = fread(buffer, 1, maxlength, (FILE *)stream);
129 *read = static_cast<unsigned int>(num);
131 if (num == 0 && feof((FILE *)stream))
136 return (ferror((FILE *)stream) == 0);
139 SMCError TextParsers::ParseFile_SMC(const char *file, ITextListener_SMC *smc, SMCStates *states)
141 FILE *fp = fopen(file, "rt");
150 return SMCError_StreamOpen;
153 SMCError result = ParseStream_SMC(fp, FileStreamReader, smc, states);
160 SMCError TextParsers::ParseSMCFile(const char *file,
161 ITextListener_SMC *smc_listener,
167 FILE *fp = fopen(file, "rt");
171 char error[256] = "unknown";
177 g_LibSys.GetPlatformError(error, sizeof(error));
178 UTIL_Format(buffer, maxsize, "File could not be opened: %s", error);
179 return SMCError_StreamOpen;
182 SMCError result = ParseStream_SMC(fp, FileStreamReader, smc_listener, states);
186 errstr = GetSMCErrorString(result);
187 UTIL_Format(buffer, maxsize, "%s", errstr != NULL ? errstr : "Unknown error");
193 * Raw parsing of streams with helper functions
198 StringInfo() : quoted(false), ptr(NULL), end(NULL), special(false) { }
205 const char *FixupString(StringInfo &data)
218 /* A string will never have beginning whitespace because we ignore it in the stream.
219 * Furthermore, if there is trailing whitespace, the end ptr will point to it, so it is valid
220 * to overwrite! Lastly, the last character must be whitespace or a comment/invalid character.
225 /* Do some extra work on strings that have special quoted characters. */
228 char *outptr = data.ptr;
229 size_t len = data.end - data.ptr;
232 for (size_t i=0; i<len; i++)
234 if (data.ptr[i] == '\\' && i < len - 1)
236 /* Resolve the next character. */
238 if (data.ptr[i] == 'n')
241 } else if (data.ptr[i] == 't') {
243 } else if (data.ptr[i] == 'r') {
245 } else if (data.ptr[i] != '\\'
246 && data.ptr[i] != '"') {
247 /* This character is invalid, so go back one */
251 *outptr++ = data.ptr[i];
265 const char *rotate(StringInfo info[3])
267 if (info[2].ptr != NULL)
272 if (info[0].ptr != NULL)
276 info[0] = StringInfo();
282 void scrap(StringInfo info[3])
284 info[2] = StringInfo();
285 info[1] = StringInfo();
286 info[0] = StringInfo();
289 void reloc(StringInfo &data, unsigned int bytes)
301 char *lowstring(StringInfo info[3])
303 for (int i=2; i>=0; i--)
314 SMCError TextParsers::ParseStream_SMC(void *stream,
316 ITextListener_SMC *smc,
319 char *reparse_point = NULL;
321 char *parse_point = in_buf;
322 char *line_begin = in_buf;
324 unsigned int curlevel = 0;
325 bool in_quote = false;
326 bool ignoring = false;
327 bool eol_comment = false;
328 bool ml_comment = false;
330 SMCError err = SMCError_Okay;
335 StringInfo strings[3];
336 StringInfo emptystring;
341 smc->ReadSMC_ParseStart();
344 * The stream reader reads in as much as it can fill the buffer with.
345 * It then processes the buffer. If the buffer cannot be fully processed, for example,
346 * a line is left hanging with no newline, then the contents of the buffer is shifted
347 * down, and the buffer is filled from the stream reader again.
349 * What makes this particularly annoying is that we cache pointers everywhere, so when
350 * the shifting process takes place, all those pointers must be shifted as well.
352 while (srdr(stream, parse_point, sizeof(in_buf) - (parse_point - in_buf) - 1, &read))
359 /* Check for BOM markings, which is only relevant on the first line.
360 * Not worth it, but it could be moved out of the loop.
362 if (states.line == 1 &&
363 in_buf[0] == (char)0xEF &&
364 in_buf[1] == (char)0xBB &&
365 in_buf[2] == (char)0xBF)
367 /* Move EVERYTHING down :\ */
368 memmove(in_buf, &in_buf[3], read - 3);
374 read += (parse_point - reparse_point);
375 parse_point = reparse_point;
376 reparse_point = NULL;
379 for (i=0; i<read; i++)
384 /* If we got a newline, there's a lot of things that could have happened in the interim.
385 * First, let's make sure the staged strings are rotated.
389 strings[0].end = &parse_point[i];
390 if (rotate(strings) != NULL)
392 err = SMCError_InvalidTokens;
397 /* Next, let's clear some line-based values that may no longer have meaning */
400 if (ignoring && !ml_comment)
405 /* Pass the raw line onto the listener. We terminate the line so the receiver
406 * doesn't get tons of useless info. We restore the newline after.
408 parse_point[i] = '\0';
409 if ((res=smc->ReadSMC_RawLine(&states, line_begin)) != SMCResult_Continue)
411 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
414 parse_point[i] = '\n';
416 /* Now we check the sanity of our staged strings! */
421 err = SMCError_InvalidProperty1;
424 /* Assume the next string is a property and pass the info on. */
425 if ((res=smc->ReadSMC_KeyValue(
427 FixupString(strings[2]),
428 FixupString(strings[1]))) != SMCResult_Continue)
430 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
436 /* Change the states for the next line */
439 line_begin = &parse_point[i+1]; //Note: safe because this gets relocated later
445 /* If i was 0, we could have reparsed, so make sure there's no buffer underrun */
446 if ((&parse_point[i] != in_buf) && c == '"' && parse_point[i-1] != '\\')
448 /* If we reached a quote in an ignore phase,
449 * we're staging a string and we must rotate it out.
454 strings[0].end = &parse_point[i];
455 strings[0].quoted = true;
456 if (rotate(strings) != NULL)
458 /* If we rotated too many strings, there was too much crap on one line */
459 err = SMCError_InvalidTokens;
465 strings[0].special = true;
468 reparse_point = &parse_point[i];
477 /* Check if we need to get more input first */
480 reparse_point = &parse_point[i];
483 if (parse_point[i+1] == '/')
487 /* We should not be staging anything right now. */
488 assert(strings[0].ptr == NULL);
489 /* Advance the input stream so we don't choke on this token */
498 /* Check if we're whitespace or not */
499 if (!g_ws_chartable[(unsigned)c])
501 bool restage = false;
502 /* Check various special tokens:
509 if (c == ';' || c == '/')
511 /* If it's a line-based comment (that is, ; or //)
512 * we will need to scrap everything until the end of the line.
518 /* If we reached the end of the look-ahead, we need to re-check our input.
519 * Breaking out will force this to be the new reparse point!
521 reparse_point = &parse_point[i];
524 if (parse_point[i + 1] == '/')
526 /* standard comment */
531 else if (parse_point[i+1] == '*')
533 /* inline comment - start ignoring */
536 /* yes, we restage, meaning that:
537 * STR/ *stuff* /ING (space because ml comments don't nest in C++)
538 * will not generate 'STRING', but rather 'STR' and 'ING'.
539 * This should be a rare occurrence and is done here for convenience.
553 /* If we are staging a string, we must rotate here */
556 /* We have unacceptable tokens on this line */
557 if (rotate(strings) != NULL)
559 err = SMCError_InvalidSection1;
563 /* Sections must always be alone */
564 if (strings[2].ptr != NULL)
566 err = SMCError_InvalidSection1;
569 else if (strings[1].ptr == NULL)
571 err = SMCError_InvalidSection2;
574 if ((res=smc->ReadSMC_NewSection(&states, FixupString(strings[1])))
575 != SMCResult_Continue)
577 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
580 strings[1] = emptystring;
585 /* Unlike our matching friend, this can be on the same line as something prior */
586 if (rotate(strings) != NULL)
588 err = SMCError_InvalidSection3;
595 err = SMCError_InvalidProperty1;
598 if ((res=smc->ReadSMC_KeyValue(
600 FixupString(strings[2]),
601 FixupString(strings[1])))
602 != SMCResult_Continue)
604 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
608 else if (strings[1].ptr)
610 err = SMCError_InvalidSection3;
615 err = SMCError_InvalidSection4;
618 /* Now it's safe to leave the section */
620 if ((res=smc->ReadSMC_LeavingSection(&states)) != SMCResult_Continue)
622 err = (res == SMCResult_HaltFail) ? SMCError_Custom : SMCError_Okay;
629 /* If we get a quote mark, we always restage, but we need to do it beforehand */
632 strings[0].end = &parse_point[i];
633 if (rotate(strings) != NULL)
635 err = SMCError_InvalidTokens;
639 strings[0].ptr = &parse_point[i];
643 else if (!strings[0].ptr)
645 /* If we have no string, we must start one */
646 strings[0].ptr = &parse_point[i];
648 if (restage && strings[0].ptr)
650 strings[0].end = &parse_point[i];
651 if (rotate(strings) != NULL)
653 err = SMCError_InvalidTokens;
660 /* If we're eating a string and get whitespace, we need to restage.
661 * (Note that if we are quoted, this is being ignored)
666 * The specification says the second string in a pair does not need to be quoted.
667 * Thus, we check if there's already a string on the stack.
668 * If there's a newline, we always rotate so the newline has an empty starter.
672 /* There's no string, so we must move this one down and eat up another */
673 strings[0].end = &parse_point[i];
676 else if (!strings[1].quoted)
678 err = SMCError_InvalidTokens;
685 /* Advance which token we're on */
689 if (line_begin != in_buf)
691 /* The line buffer has advanced, so it's safe to copy N bytes back to the beginning.
692 * What's N? N is the lowest point we're currently relying on.
694 char *stage = lowstring(strings);
695 if (!stage || stage > line_begin)
699 unsigned int bytes = read - (stage - parse_point);
701 /* It is now safe to delete everything before the staged point */
702 memmove(in_buf, stage, bytes);
704 /* Calculate the number of bytes in the new buffer */
705 bytes = stage - in_buf;
706 /* Relocate all the cached pointers to our new base */
708 reloc(strings[0], bytes);
709 reloc(strings[1], bytes);
710 reloc(strings[2], bytes);
713 reparse_point -= bytes;
717 parse_point = &parse_point[read];
718 parse_point -= bytes;
721 else if (read == sizeof(in_buf) - 1)
723 err = SMCError_TokenOverflow;
728 /* If we're done parsing and there are tokens left over... */
731 err = SMCError_InvalidSection5;
734 else if (strings[0].ptr || strings[1].ptr)
736 err = SMCError_InvalidTokens;
740 smc->ReadSMC_ParseEnd(false, false);
747 return SMCError_Okay;
755 smc->ReadSMC_ParseEnd(true, (err == SMCError_Custom));
765 bool TextParsers::ParseFile_INI(const char *file, ITextListener_INI *ini_listener, unsigned int *line, unsigned int *col)
767 FILE *fp = fopen(file, "rt");
768 unsigned int curline = 0;
783 char *ptr, *save_ptr;
791 if (fgets(buffer, sizeof(buffer), fp) == NULL)
796 //:TODO: this will only run once, so find a nice way to move it out of the while loop
797 /* If this is the first line, check the first three bytes for BOM */
799 buffer[0] == (char)0xEF &&
800 buffer[1] == (char)0xBB &&
801 buffer[2] == (char)0xBF)
803 /* We have a UTF-8 marked file... skip these bytes */
809 /***************************************************
810 * We preprocess the string before parsing tokens! *
811 ***************************************************/
813 /* First strip beginning whitespace */
814 while (*ptr != '\0' && g_ws_chartable[(unsigned)*ptr] != 0)
826 /* Now search for comment characters */
829 for (size_t i=0; i<len; i++,ptr++)
844 /* Terminate the string here */
864 /* Lastly, strip ending whitespace off */
865 for (size_t i=len-1; i>=0 && i<len; i--)
867 if (g_ws_chartable[(unsigned)ptr[i]])
881 if (!ini_listener->ReadINI_RawLine(ptr, &curtok))
888 bool invalid_tokens = false;
889 bool got_bracket = false;
890 bool extra_tokens = false;
895 for (size_t i=1; i<len; i++)
902 if (mbtowc(&wc, &ptr[i], len-i) != -1)
904 alnum = (iswalnum(wc) != 0);
905 i += _GetUTF8CharBytes(&ptr[i]) - 1;
908 alnum = (isalnum(c) != 0) || (g_ini_chartable1[(unsigned)c] != 0);
912 /* First check - is this a bracket? */
917 /* If this isn't the last character... */
926 /* n...No! Continue copying. */
927 invalid_tokens = true;
932 /* Tell the handler */
933 if (!ini_listener->ReadINI_NewSection(&ptr[1], invalid_tokens, got_bracket, extra_tokens, &curtok))
939 char *val_ptr = NULL;
941 size_t first_space = 0;
942 bool invalid_tokens = false;
943 bool equal_token = false;
948 for (size_t i=0; i<len; i++)
952 /* is this an invalid char? */
955 if (mbtowc(&wc, &ptr[i], len-i) != -1)
957 alnum = (iswalnum(wc) != 0);
958 i += _GetUTF8CharBytes(&ptr[i]) - 1;
961 alnum = (isalnum(c) != 0) || (g_ini_chartable1[(unsigned)c] != 0);
966 if (g_ws_chartable[(unsigned)c])
968 /* if it's a space, keep track of the first occurring space */
976 /* if it's an equal sign, we're done with the key */
979 /* remove excess whitespace */
980 key_ptr[first_space] = '\0';
982 /* remove the equal sign */
985 if (ptr[++i] != '\0')
987 /* If this isn't the end, set next pointer */
993 /* Mark that we got something invalid! */
994 invalid_tokens = true;
1001 /* Now we need to parse the value, if any */
1004 /* eat up spaces! there shouldn't be any h*/
1005 while ((*val_ptr != '\0') && g_ws_chartable[(unsigned)*val_ptr] != 0)
1009 if (*val_ptr == '\0')
1014 /* Do we have an initial quote? If so, the parsing rules change! */
1015 if (*val_ptr == '"' && *val_ptr != '\0')
1017 len = strlen(val_ptr);
1018 if (val_ptr[len-1] == '"')
1021 val_ptr[--len] = '\0';
1029 curtok = val_ptr - buffer;
1030 if (!ini_listener->ReadINI_KeyValue(key_ptr, val_ptr, invalid_tokens, equal_token, quotes, &curtok))
1063 const char *TextParsers::GetSMCErrorString(SMCError err)
1065 static const char *s_errors[] =
1068 "Stream failed to open",
1069 "Stream returned read error",
1071 "Un-quoted section has invalid tokens",
1072 "Section declared without header",
1073 "Section declared with unknown tokens",
1074 "Section ending without a matching section beginning",
1075 "Section beginning without a matching ending",
1076 "Line contained too many invalid tokens",
1077 "Token buffer overflowed",
1078 "A property was declared outside of a section",
1081 if (err < SMCError_Okay || err > SMCError_InvalidProperty1)
1086 return s_errors[err];