4 #include "CTranslator.h"
6 #include "CTextParsers.h"
7 #include "LibrarySys.h"
8 #include "sm_stringutil.h"
11 CTranslator g_Translator;
22 unsigned int fmt_count;
23 unsigned int fmt_bytes;
25 unsigned int translations;
28 CPhraseFile::CPhraseFile(CTranslator *pTranslator, const char *file)
30 m_pStringTab = pTranslator->GetStringTable();
31 m_pMemory = m_pStringTab->GetMemTable();
32 m_LangCount = pTranslator->GetLanguageCount();
34 m_pTranslator = pTranslator;
35 m_pPhraseLookup = NULL;
38 CPhraseFile::~CPhraseFile()
42 sm_trie_destroy(m_pPhraseLookup);
46 void CPhraseFile::ParseError(const char *message, ...)
50 va_start(ap, message);
51 vsnprintf(buffer, sizeof(buffer), message, ap);
54 m_ParseError.assign(buffer);
57 void CPhraseFile::ParseWarning(const char *message, ...)
61 va_start(ap, message);
62 vsnprintf(buffer, sizeof(buffer), message, ap);
67 g_Logger.LogError("[SM] Warning(s) encountered in translation file \"%s\"", m_File.c_str());
71 g_Logger.LogError("[SM] %s", message);
74 void CPhraseFile::ReparseFile()
78 sm_trie_destroy(m_pPhraseLookup);
80 m_pPhraseLookup = sm_trie_create();
82 m_LangCount = m_pTranslator->GetLanguageCount();
90 char path[PLATFORM_MAX_PATH+1];
91 g_SourceMod.BuildPath(Path_SM, path, PLATFORM_MAX_PATH, "translations/%s", m_File.c_str());
92 unsigned int line=0, col=0;
94 if ((err=g_TextParser.ParseFile_SMC(path, this, &line, &col)) != SMCParse_Okay)
96 const char *msg = g_TextParser.GetSMCErrorString(err);
99 msg = m_ParseError.c_str();
102 g_Logger.LogError("[SM] Fatal error encountered parsing translation file \"%s\"", m_File.c_str());
103 g_Logger.LogError("[SM] Error (line %d, column %d): %s", line, col, msg);
107 void CPhraseFile::ReadSMC_ParseStart()
110 m_ParseState = PPS_None;
112 m_FileLogged = false;
113 m_LastPhraseString.clear();
116 SMCParseResult CPhraseFile::ReadSMC_RawLine(const char *line, unsigned int curline)
120 return SMCParse_Continue;
123 SMCParseResult CPhraseFile::ReadSMC_NewSection(const char *name, bool opt_quotes)
125 bool recognized = false;
126 if (m_ParseState == PPS_None)
128 if (strcmp(name, "Phrases") == 0)
130 m_ParseState = PPS_Phrases;
133 } else if (m_ParseState == PPS_Phrases) {
134 m_ParseState = PPS_InPhrase;
138 m_CurPhrase = m_pMemory->CreateMem(sizeof(phrase_t), (void **)&pPhrase);
140 /* Create the reverse lookup */
141 if (!sm_trie_insert(m_pPhraseLookup, name, reinterpret_cast<void *>(m_CurPhrase)))
143 ParseWarning("Skipping duplicate phrase \"%s\" on line %d.", name, m_CurLine);
144 /* :IDEA: prevent memory waste by seeking backwards in memtable? */
147 /* Initialize new phrase */
150 pPhrase->fmt_count = 0;
151 pPhrase->fmt_list = -1;
152 pPhrase->trans_tbl = m_pMemory->CreateMem(sizeof(trans_t) * m_LangCount, (void **)&pTrans);
153 pPhrase->translations = 0;
154 pPhrase->fmt_bytes = 0;
156 for (unsigned int i=0; i<m_LangCount; i++)
158 pTrans[i].stridx = -1;
160 m_LastPhraseString.assign(name);
162 } else if (m_ParseState == PPS_InPhrase) {
163 ParseError("Phrase sections may not have sub-sections");
164 return SMCParse_HaltFail;
169 ParseWarning("Ignoring invalid section \"%s\" on line %d.", name, m_CurLine);
172 return SMCParse_Continue;
175 SMCParseResult CPhraseFile::ReadSMC_KeyValue(const char *key, const char *value, bool key_quotes, bool value_quotes)
177 /* See if we are ignoring a phrase */
180 return SMCParse_Continue;
183 phrase_t *pPhrase = (phrase_t *)m_pMemory->GetAddress(m_CurPhrase);
185 if (key[0] == '#' && strcmp(key, "#format") == 0)
187 if (pPhrase->fmt_list != -1)
189 ParseWarning("Ignoring duplicated #format property on line %d", m_CurLine);
190 return SMCParse_Continue;
193 if (pPhrase->translations > 0)
195 ParseWarning("#format property should come before translations on line %d, ignoring", m_CurLine);
196 return SMCParse_Continue;
199 /* Do an initial browsing for error checking and what not */
207 ParseStates state = Parse_None;
209 const char *old_value = value;
210 while (*value != '\0')
212 if (state == Parse_None)
216 pPhrase->fmt_count++;
218 } else if (*value == ',') {
221 unsigned int bytes = g_TextParser.GetUTF8CharBytes(value);
222 if (bytes != 1 || !isalpha(*value))
224 ParseWarning("Invalid token '%c' in #format property on line %d.", *value, m_CurLine);
227 } else if (state == Parse_Index) {
230 state = Parse_Format;
231 if (value - old_value >= 15)
233 ParseWarning("Too many digits in format index on line %d, phrase will be ignored.", m_CurLine);
235 return SMCParse_Continue;
238 unsigned int bytes = g_TextParser.GetUTF8CharBytes(value);
239 if (bytes != 1 || !isdigit(*value))
241 ParseWarning("Token '%c' in #format property on line %d is not a digit, phrase will be ignored.",
245 return SMCParse_Continue;
248 } else if (state == Parse_Format) {
257 if (state != Parse_None)
259 /* Moose clam cow. */
260 ParseWarning("Unterminated format string on line %d, phrase will be ignored.", m_CurLine);
262 return SMCParse_Continue;
266 /* Allocate the format table */
269 pPhrase->fmt_list = m_pMemory->CreateMem(sizeof(int) * pPhrase->fmt_count, (void **)&fmt_list);
272 for (size_t i=0; i<pPhrase->fmt_count; i++)
277 /* Now we need to read again... this time into the format buffer */
278 const char *in_ptr = value;
279 const char *idx_ptr = NULL;
280 char *out_ptr = NULL;
281 unsigned int cur_idx = 0;
283 while (*in_ptr != '\0')
285 if (state == Parse_None)
292 } else if (state == Parse_Index) {
295 /* Check the number! */
298 ParseWarning("Format property contains unindexed format string on line %d, phrase will be ignored.", m_CurLine);
300 return SMCParse_Continue;
302 long idx = strtol(idx_ptr, NULL, 10);
303 if (idx < 1 || idx > (long)pPhrase->fmt_count)
305 ParseWarning("Format property contains invalid index '%d' on line %d, phrase will be ignored.", idx, m_CurLine);
307 return SMCParse_Continue;
308 } else if (fmt_list[idx - 1] != -1) {
309 ParseWarning("Format property contains duplicated index '%d' on line %d, phrase will be ignored.", idx, m_CurLine);
311 return SMCParse_Continue;
313 cur_idx = (unsigned int)idx;
314 state = Parse_Format;
316 } else if (!idx_ptr) {
319 } else if (state == Parse_Format) {
324 ParseWarning("Format property contains empty format string on line %d, phrase will be ignored.", m_CurLine);
326 return SMCParse_Continue;
330 /* Now, add this to our table */
331 fmt_list[cur_idx - 1] = m_pStringTab->AddString(fmt_buf);
332 pPhrase->fmt_bytes += strlen(fmt_buf);
339 /* Check length ... */
340 if ((unsigned)(out_ptr - fmt_buf) >= sizeof(fmt_buf) - 1)
342 ParseWarning("Format property contains format string that exceeds maximum length on line %d, phrase will be ignored.",
345 return SMCParse_Continue;
347 *out_ptr++ = *in_ptr;
353 /* If we've gotten here, we only accepted unique indexes in a range.
354 * Therefore, the list has to be completely filled. Double check anyway.
356 for (size_t i=0; i<pPhrase->fmt_count; i++)
358 if (fmt_list[i] == -1)
360 ParseWarning("Format property contains no string for index %d on line %d, phrase will be ignored.",
364 return SMCParse_Continue;
368 size_t len = strlen(key);
371 ParseWarning("Ignoring translation to invalid language \"%s\" on line %d.", key, m_CurLine);
372 return SMCParse_Continue;
376 if (!m_pTranslator->GetLanguageByCode(key, &lang))
378 /* Ignore if we don't have a language.
379 * :IDEA: issue a one-time alert?
381 return SMCParse_Continue;
384 /* See how many bytes we need for this string, then allocate.
385 * NOTE: THIS SHOULD GUARANTEE THAT WE DO NOT NEED TO NEED TO SIZE CHECK
387 len = strlen(value) + pPhrase->fmt_bytes + 1;
391 out_idx = m_pMemory->CreateMem(len, (void **)&out_buf);
394 int *fmt_list = (int *)m_pMemory->GetAddress(pPhrase->fmt_list);
395 trans_t *pTrans = (trans_t *)m_pMemory->GetAddress(pPhrase->trans_tbl);
397 pTrans = &pTrans[lang];
398 pTrans->stridx = out_idx;
400 bool params[MAX_TRANSLATE_PARAMS];
402 /* Build the format order list, if necessary */
405 pTrans->fmt_order = m_pMemory->CreateMem(pPhrase->fmt_count * sizeof(int), (void **)&fmt_order);
407 for (unsigned int i=0; i<pPhrase->fmt_count; i++)
411 memset(¶ms[0], 0, sizeof(params));
414 const char *in_ptr = value;
415 char *out_ptr = out_buf;
416 unsigned int order_idx = 0;
417 while (*in_ptr != '\0')
421 switch (*(in_ptr + 1))
436 /* Copy both bytes since we don't know what's going on */
437 *out_ptr++ = *in_ptr++;
438 *out_ptr++ = *in_ptr;
441 /* Skip past the last byte read */
443 } else if (*in_ptr == '{' && fmt_list != NULL) {
444 /* Search for parameters if this is a formatted string */
445 const char *scrap_in_point = in_ptr;
446 const char *digit_start = ++in_ptr;
448 while (*in_ptr != '\0')
450 bytes = g_TextParser.GetUTF8CharBytes(in_ptr);
457 /* Did we get an index? */
458 if (in_ptr == digit_start)
463 long idx = strtol(digit_start, NULL, 10);
464 if (idx < 1 || idx > (int)pPhrase->fmt_count)
473 /* We're safe to insert the string. First mark order. */
474 fmt_order[order_idx++] = (int)idx - 1;
475 /* Now concatenate */
476 out_ptr += sprintf(out_ptr, "%s", m_pStringTab->GetString(fmt_list[idx-1]));
478 params[idx-1] = true;
485 /* Pretend none of this ever happened. Move along now! */
486 in_ptr = scrap_in_point;
487 *out_ptr++ = *in_ptr;
489 *out_ptr++ = *in_ptr;
495 pPhrase->translations++;
498 return SMCParse_Continue;
501 SMCParseResult CPhraseFile::ReadSMC_LeavingSection()
503 if (m_ParseState == PPS_InPhrase)
505 if (m_CurPhrase == -1 && m_LastPhraseString.size())
507 sm_trie_delete(m_pPhraseLookup, m_LastPhraseString.c_str());
510 m_ParseState = PPS_Phrases;
511 m_LastPhraseString.assign("");
512 } else if (m_ParseState == PPS_Phrases) {
513 m_ParseState = PPS_None;
516 return SMCParse_Continue;
519 void CPhraseFile::ReadSMC_ParseEnd(bool halted, bool failed)
521 /* Check to see if we have any dangling phrases that weren't completed, and scrap them */
522 if ((halted || failed) && m_LastPhraseString.size())
524 sm_trie_delete(m_pPhraseLookup, m_LastPhraseString.c_str());
528 const char *CPhraseFile::GetTranslation(const Translation *pTrans, int **fmt_order, unsigned int *fmt_count)
530 if (pTrans->lang_id >= m_LangCount)
536 if (!sm_trie_retrieve(m_pPhraseLookup, pTrans->szPhrase, &object))
541 phrase_t *pPhrase = (phrase_t *)m_pMemory->GetAddress(reinterpret_cast<int>(object));
542 trans_t *trans = (trans_t *)m_pMemory->GetAddress(pPhrase->trans_tbl);
544 trans = &trans[pTrans->lang_id];
546 if (trans->stridx == -1)
551 *fmt_order = (int *)m_pMemory->GetAddress(trans->fmt_order);
552 *fmt_count = pPhrase->fmt_count;
554 return m_pStringTab->GetString(trans->stridx);
557 const char *CPhraseFile::GetFilename()
559 return m_File.c_str();
562 CTranslator::CTranslator()
564 m_pStringTab = new BaseStringTable(2048);
565 m_pLCodeLookup = sm_trie_create();
568 CTranslator::~CTranslator()
570 for (size_t i=0; i<m_Files.size(); i++)
575 for (size_t i=0; i<m_Languages.size(); i++)
577 delete m_Languages[i];
580 sm_trie_destroy(m_pLCodeLookup);
585 bool CTranslator::GetLanguageByCode(const char *code, unsigned int *index)
589 if (!sm_trie_retrieve(m_pLCodeLookup, code, &_index))
596 *index = reinterpret_cast<unsigned int>(_index);
602 unsigned int CTranslator::GetLanguageCount()
604 return (unsigned int)m_Languages.size();
607 BaseStringTable *CTranslator::GetStringTable()
612 unsigned int CTranslator::FindOrAddPhraseFile(const char *phrase_file)
614 for (size_t i=0; i<m_Files.size(); i++)
616 if (strcmp(m_Files[i]->GetFilename(), phrase_file) == 0)
618 return (unsigned int)i;
622 CPhraseFile *pFile = new CPhraseFile(this, phrase_file);
623 unsigned int idx = (unsigned int)m_Files.size();
625 m_Files.push_back(pFile);
627 pFile->ReparseFile();
632 void CTranslator::RebuildLanguageDatabase(const char *lang_header_file)
634 /* Erase everything we have */
635 sm_trie_destroy(m_pLCodeLookup);
636 m_pLCodeLookup = sm_trie_create();
637 m_pStringTab->Reset();
639 for (size_t i=0; i<m_Languages.size(); i++)
641 delete m_Languages[i];
647 unsigned int line=0, col=0;
648 if ((err=g_TextParser.ParseFile_SMC(lang_header_file, this, &line, &col)) != SMCParse_Okay)
650 const char *str_err = g_TextParser.GetSMCErrorString(err);
653 str_err = m_CustomError.c_str();
656 g_Logger.LogError("[SM] Failed to parse language header file: \"%s\"", lang_header_file);
657 g_Logger.LogError("[SM] Parse error (line %d, column %d): %s", line, col, str_err);
660 if (!m_Languages.size())
662 g_Logger.LogError("[SM] Fatal error, no languages found! Translation will not work.");
665 for (size_t i=0; i<m_Files.size(); i++)
667 m_Files[i]->ReparseFile();
671 void CTranslator::ReadSMC_ParseStart()
673 m_InLanguageSection = false;
674 m_CustomError.clear();
677 SMCParseResult CTranslator::ReadSMC_NewSection(const char *name, bool opt_quotes)
679 if (!m_InLanguageSection)
681 if (strcmp(name, "Languages") == 0)
683 m_InLanguageSection = true;
687 if (!m_InLanguageSection)
689 g_Logger.LogError("[SM] Warning: Unrecognized section \"%s\" in languages.cfg", name);
692 return SMCParse_Continue;
695 SMCParseResult CTranslator::ReadSMC_LeavingSection()
697 m_InLanguageSection = false;
699 return SMCParse_Continue;
702 SMCParseResult CTranslator::ReadSMC_KeyValue(const char *key, const char *value, bool key_quotes, bool value_quotes)
704 size_t len = strlen(key);
708 g_Logger.LogError("[SM] Warning encountered parsing languages.cfg file.");
709 g_Logger.LogError("[SM] Invalid language code \"%s\" is being ignored.", key);
712 Language *pLanguage = new Language;
713 unsigned int idx = m_Languages.size();
715 strcpy(pLanguage->m_code2, key);
716 pLanguage->m_FullName = m_pStringTab->AddString(value);
718 sm_trie_insert(m_pLCodeLookup, key, reinterpret_cast<void *>(idx));
720 m_Languages.push_back(pLanguage);
722 return SMCParse_Continue;
725 size_t CTranslator::Translate(char *buffer, size_t maxlength, const Translation *pTrans)
727 if (pTrans->file_id >= m_Files.size())
732 CPhraseFile *pFile = m_Files[pTrans->file_id];
737 if ((str=pFile->GetTranslation(pTrans, &order_table, &count)) == NULL)
742 void *params[MAX_TRANSLATE_PARAMS];
744 /* Rewrite the parameter order */
745 for (unsigned int i=0; i<count; i++)
747 params[i] = pTrans->params[order_table[i]];
750 return gnprintf(buffer, maxlength, str, params);