3 * ===============================================================
4 * SourceMod (C)2004-2007 AlliedModders LLC. All rights reserved.
5 * ===============================================================
7 * This file is not open source and may not be copied without explicit
8 * written permission of AlliedModders LLC. This file may not be redistributed
9 * in whole or significant part.
10 * For information, see LICENSE.txt or http://www.sourcemod.net/license.php
18 #include "Translator.h"
20 #include "TextParsers.h"
21 #include "LibrarySys.h"
22 #include "sm_stringutil.h"
23 #include "sourcemod.h"
25 Translator g_Translator;
26 CPhraseFile *g_pCorePhrases = NULL;
37 unsigned int fmt_count;
38 unsigned int fmt_bytes;
40 unsigned int translations;
43 CPhraseFile::CPhraseFile(Translator *pTranslator, const char *file)
45 m_pStringTab = pTranslator->GetStringTable();
46 m_pMemory = m_pStringTab->GetMemTable();
47 m_LangCount = pTranslator->GetLanguageCount();
49 m_pTranslator = pTranslator;
50 m_pPhraseLookup = NULL;
53 CPhraseFile::~CPhraseFile()
57 sm_trie_destroy(m_pPhraseLookup);
61 void CPhraseFile::ParseError(const char *message, ...)
65 va_start(ap, message);
66 vsnprintf(buffer, sizeof(buffer), message, ap);
69 m_ParseError.assign(buffer);
72 void CPhraseFile::ParseWarning(const char *message, ...)
76 va_start(ap, message);
77 vsnprintf(buffer, sizeof(buffer), message, ap);
82 g_Logger.LogError("[SM] Warning(s) encountered in translation file \"%s\"", m_File.c_str());
86 g_Logger.LogError("[SM] %s", message);
89 void CPhraseFile::ReparseFile()
93 sm_trie_destroy(m_pPhraseLookup);
95 m_pPhraseLookup = sm_trie_create();
97 m_LangCount = m_pTranslator->GetLanguageCount();
105 char path[PLATFORM_MAX_PATH];
106 g_SourceMod.BuildPath(Path_SM, path, PLATFORM_MAX_PATH, "translations/%s", m_File.c_str());
108 //backwards compatibility shim
109 /* :HACKHACK: Change .cfg/.txt and vice versa for compatibility */
110 if (!g_LibSys.PathExists(path))
112 if (m_File.compare("common.cfg") == 0)
114 UTIL_ReplaceAll(path, sizeof(path), "common.cfg", "common.phrases.txt");
115 } else if (strstr(path, ".cfg")) {
116 UTIL_ReplaceAll(path, sizeof(path), ".cfg", ".txt");
117 } else if (strstr(path, ".txt")) {
118 UTIL_ReplaceAll(path, sizeof(path), ".txt", ".cfg");
122 unsigned int line=0, col=0;
124 if ((err=g_TextParser.ParseFile_SMC(path, this, &line, &col)) != SMCParse_Okay)
126 const char *msg = g_TextParser.GetSMCErrorString(err);
129 msg = m_ParseError.c_str();
132 g_Logger.LogError("[SM] Fatal error encountered parsing translation file \"%s\"", m_File.c_str());
133 g_Logger.LogError("[SM] Error (line %d, column %d): %s", line, col, msg);
137 void CPhraseFile::ReadSMC_ParseStart()
140 m_ParseState = PPS_None;
142 m_FileLogged = false;
143 m_LastPhraseString.clear();
146 SMCParseResult CPhraseFile::ReadSMC_RawLine(const char *line, unsigned int curline)
150 return SMCParse_Continue;
153 SMCParseResult CPhraseFile::ReadSMC_NewSection(const char *name, bool opt_quotes)
155 bool recognized = false;
156 if (m_ParseState == PPS_None)
158 if (strcmp(name, "Phrases") == 0)
160 m_ParseState = PPS_Phrases;
163 } else if (m_ParseState == PPS_Phrases) {
164 m_ParseState = PPS_InPhrase;
168 m_CurPhrase = m_pMemory->CreateMem(sizeof(phrase_t), (void **)&pPhrase);
170 /* Create the reverse lookup */
171 if (!sm_trie_insert(m_pPhraseLookup, name, reinterpret_cast<void *>(m_CurPhrase)))
173 ParseWarning("Skipping duplicate phrase \"%s\" on line %d.", name, m_CurLine);
174 /* :IDEA: prevent memory waste by seeking backwards in memtable? */
177 /* Initialize new phrase */
180 pPhrase->fmt_count = 0;
181 pPhrase->fmt_list = -1;
183 int trans_tbl = m_pMemory->CreateMem(sizeof(trans_t) * m_LangCount, (void **)&pTrans);
184 /* Update the pointer! */
185 pPhrase = (phrase_t *)m_pMemory->GetAddress(m_CurPhrase);
186 pPhrase->trans_tbl = trans_tbl;
188 pPhrase->translations = 0;
189 pPhrase->fmt_bytes = 0;
191 for (unsigned int i=0; i<m_LangCount; i++)
193 pTrans[i].stridx = -1;
195 m_LastPhraseString.assign(name);
197 } else if (m_ParseState == PPS_InPhrase) {
198 ParseError("Phrase sections may not have sub-sections");
199 return SMCParse_HaltFail;
204 ParseWarning("Ignoring invalid section \"%s\" on line %d.", name, m_CurLine);
207 return SMCParse_Continue;
210 SMCParseResult CPhraseFile::ReadSMC_KeyValue(const char *key, const char *value, bool key_quotes, bool value_quotes)
212 /* See if we are ignoring a phrase */
213 if (m_CurPhrase == -1)
215 return SMCParse_Continue;
218 phrase_t *pPhrase = (phrase_t *)m_pMemory->GetAddress(m_CurPhrase);
220 if (key[0] == '#' && strcmp(key, "#format") == 0)
222 if (pPhrase->fmt_list != -1)
224 ParseWarning("Ignoring duplicated #format property on line %d", m_CurLine);
225 return SMCParse_Continue;
228 if (pPhrase->translations > 0)
230 ParseWarning("#format property should come before translations on line %d, ignoring", m_CurLine);
231 return SMCParse_Continue;
234 /* Do an initial browsing for error checking and what not */
242 ParseStates state = Parse_None;
244 const char *old_value = value;
245 const char *last_value = value;
246 while (*value != '\0')
248 if (state == Parse_None)
252 pPhrase->fmt_count++;
254 } else if (*value == ',') {
257 unsigned int bytes = g_TextParser.GetUTF8CharBytes(value);
258 if (bytes != 1 || !isalpha(*value))
260 ParseWarning("Invalid token '%c' in #format property on line %d.", *value, m_CurLine);
263 } else if (state == Parse_Index) {
266 state = Parse_Format;
267 if (value - last_value >= 15)
269 ParseWarning("Too many digits in format index on line %d, phrase will be ignored.", m_CurLine);
271 return SMCParse_Continue;
274 unsigned int bytes = g_TextParser.GetUTF8CharBytes(value);
275 if (bytes != 1 || !isdigit(*value))
277 ParseWarning("Token '%c' in #format property on line %d is not a digit, phrase will be ignored.",
281 return SMCParse_Continue;
284 } else if (state == Parse_Format) {
288 last_value = value + 1;
294 if (state != Parse_None)
296 /* Moose clam cow. */
297 ParseWarning("Unterminated format string on line %d, phrase will be ignored.", m_CurLine);
299 return SMCParse_Continue;
303 /* Allocate the format table */
306 pPhrase->fmt_list = m_pMemory->CreateMem(sizeof(int) * pPhrase->fmt_count, (void **)&fmt_list);
309 for (size_t i=0; i<pPhrase->fmt_count; i++)
314 /* Now we need to read again... this time into the format buffer */
315 const char *in_ptr = value;
316 const char *idx_ptr = NULL;
317 char *out_ptr = NULL;
318 unsigned int cur_idx = 0;
320 while (*in_ptr != '\0')
322 if (state == Parse_None)
329 } else if (state == Parse_Index) {
332 /* Check the number! */
335 ParseWarning("Format property contains unindexed format string on line %d, phrase will be ignored.", m_CurLine);
337 return SMCParse_Continue;
339 long idx = strtol(idx_ptr, NULL, 10);
340 if (idx < 1 || idx > (long)pPhrase->fmt_count)
342 ParseWarning("Format property contains invalid index '%d' on line %d, phrase will be ignored.", idx, m_CurLine);
344 return SMCParse_Continue;
345 } else if (fmt_list[idx - 1] != -1) {
346 ParseWarning("Format property contains duplicated index '%d' on line %d, phrase will be ignored.", idx, m_CurLine);
348 return SMCParse_Continue;
350 cur_idx = (unsigned int)idx;
351 state = Parse_Format;
353 } else if (!idx_ptr) {
356 } else if (state == Parse_Format) {
361 ParseWarning("Format property contains empty format string on line %d, phrase will be ignored.", m_CurLine);
363 return SMCParse_Continue;
367 /* Now, add this to our table */
368 fmt_list[cur_idx - 1] = m_pStringTab->AddString(fmt_buf);
369 pPhrase->fmt_bytes += strlen(fmt_buf);
376 /* Check length ... */
377 if ((unsigned)(out_ptr - fmt_buf) >= sizeof(fmt_buf) - 1)
379 ParseWarning("Format property contains format string that exceeds maximum length on line %d, phrase will be ignored.",
382 return SMCParse_Continue;
384 *out_ptr++ = *in_ptr;
390 /* If we've gotten here, we only accepted unique indexes in a range.
391 * Therefore, the list has to be completely filled. Double check anyway.
393 for (size_t i=0; i<pPhrase->fmt_count; i++)
395 if (fmt_list[i] == -1)
397 ParseWarning("Format property contains no string for index %d on line %d, phrase will be ignored.",
401 return SMCParse_Continue;
405 size_t len = strlen(key);
408 ParseWarning("Ignoring translation to invalid language \"%s\" on line %d.", key, m_CurLine);
409 return SMCParse_Continue;
413 if (!m_pTranslator->GetLanguageByCode(key, &lang))
415 /* Ignore if we don't have a language.
416 * :IDEA: issue a one-time alert?
418 return SMCParse_Continue;
421 /* See how many bytes we need for this string, then allocate.
422 * NOTE: THIS SHOULD GUARANTEE THAT WE DO NOT NEED TO NEED TO SIZE CHECK
424 len = strlen(value) + pPhrase->fmt_bytes + 1;
428 out_idx = m_pMemory->CreateMem(len, (void **)&out_buf);
431 int *fmt_list = (int *)m_pMemory->GetAddress(pPhrase->fmt_list);
432 trans_t *pTrans = (trans_t *)m_pMemory->GetAddress(pPhrase->trans_tbl);
434 pTrans = &pTrans[lang];
435 pTrans->stridx = out_idx;
437 bool params[MAX_TRANSLATE_PARAMS];
439 /* Build the format order list, if necessary */
442 pTrans->fmt_order = m_pMemory->CreateMem(pPhrase->fmt_count * sizeof(int), (void **)&fmt_order);
444 for (unsigned int i=0; i<pPhrase->fmt_count; i++)
448 memset(¶ms[0], 0, sizeof(params));
451 const char *in_ptr = value;
452 char *out_ptr = out_buf;
453 unsigned int order_idx = 0;
454 while (*in_ptr != '\0')
458 switch (*(in_ptr + 1))
473 /* Copy both bytes since we don't know what's going on */
474 *out_ptr++ = *in_ptr++;
475 *out_ptr++ = *in_ptr;
478 /* Skip past the last byte read */
480 } else if (*in_ptr == '{' && fmt_list != NULL) {
481 /* Search for parameters if this is a formatted string */
482 const char *scrap_in_point = in_ptr;
483 const char *digit_start = ++in_ptr;
485 while (*in_ptr != '\0')
487 bytes = g_TextParser.GetUTF8CharBytes(in_ptr);
494 /* Did we get an index? */
495 if (in_ptr == digit_start)
500 long idx = strtol(digit_start, NULL, 10);
501 if (idx < 1 || idx > (int)pPhrase->fmt_count)
510 /* We're safe to insert the string. First mark order. */
511 fmt_order[order_idx++] = (int)idx - 1;
512 /* Now concatenate */
513 out_ptr += sprintf(out_ptr, "%s", m_pStringTab->GetString(fmt_list[idx-1]));
515 params[idx-1] = true;
522 /* Pretend none of this ever happened. Move along now! */
523 in_ptr = scrap_in_point;
524 *out_ptr++ = *in_ptr;
526 *out_ptr++ = *in_ptr;
532 pPhrase->translations++;
535 return SMCParse_Continue;
538 SMCParseResult CPhraseFile::ReadSMC_LeavingSection()
540 if (m_ParseState == PPS_InPhrase)
542 if (m_CurPhrase == -1 && m_LastPhraseString.size())
544 sm_trie_delete(m_pPhraseLookup, m_LastPhraseString.c_str());
547 m_ParseState = PPS_Phrases;
548 m_LastPhraseString.assign("");
549 } else if (m_ParseState == PPS_Phrases) {
550 m_ParseState = PPS_None;
553 return SMCParse_Continue;
556 void CPhraseFile::ReadSMC_ParseEnd(bool halted, bool failed)
558 /* Check to see if we have any dangling phrases that weren't completed, and scrap them */
559 if ((halted || failed) && m_LastPhraseString.size())
561 sm_trie_delete(m_pPhraseLookup, m_LastPhraseString.c_str());
565 TransError CPhraseFile::GetTranslation(const char *szPhrase, unsigned int lang_id, Translation *pTrans)
567 if (lang_id >= m_LangCount)
569 return Trans_BadLanguage;
573 if (!sm_trie_retrieve(m_pPhraseLookup, szPhrase, &object))
575 return Trans_BadPhrase;
578 phrase_t *pPhrase = (phrase_t *)m_pMemory->GetAddress(reinterpret_cast<int>(object));
579 trans_t *trans = (trans_t *)m_pMemory->GetAddress(pPhrase->trans_tbl);
581 trans = &trans[lang_id];
583 if (trans->stridx == -1)
585 return Trans_BadPhraseLanguage;
588 pTrans->fmt_order = (int *)m_pMemory->GetAddress(trans->fmt_order);
589 pTrans->fmt_count = pPhrase->fmt_count;
591 pTrans->szPhrase = m_pStringTab->GetString(trans->stridx);
596 const char *CPhraseFile::GetFilename()
598 return m_File.c_str();
601 /**************************
602 ** MAIN TRANSLATOR CODE **
603 **************************/
605 Translator::Translator() : m_ServerLang(LANGUAGE_ENGLISH)
607 m_pStringTab = new BaseStringTable(2048);
608 m_pLCodeLookup = sm_trie_create();
609 strncopy(m_InitialLang, "en", sizeof(m_InitialLang));
612 Translator::~Translator()
614 for (size_t i=0; i<m_Files.size(); i++)
619 for (size_t i=0; i<m_Languages.size(); i++)
621 delete m_Languages[i];
624 sm_trie_destroy(m_pLCodeLookup);
629 ConfigResult Translator::OnSourceModConfigChanged(const char *key,
635 if (strcasecmp(key, "ServerLang") == 0)
637 if (source == ConfigSource_Console)
640 if (!GetLanguageByCode(value, &index))
642 UTIL_Format(error, maxlength, "Language code \"%s\" is not registered", value);
643 return ConfigResult_Reject;
646 m_ServerLang = index;
648 strncopy(m_InitialLang, value, sizeof(m_InitialLang));
651 return ConfigResult_Accept;
654 return ConfigResult_Ignore;
657 void Translator::OnSourceModLevelChange(const char *mapName)
659 /* Refresh language stuff */
660 char path[PLATFORM_MAX_PATH];
661 g_SourceMod.BuildPath(Path_SM, path, sizeof(path), "configs/languages.cfg");
662 RebuildLanguageDatabase(path);
665 void Translator::OnSourceModAllInitialized()
667 AddLanguage("en", "English");
671 //backwards compatibility shim
672 /* hack -- if core.cfg exists, exec it instead. */
673 char path[PLATFORM_MAX_PATH];
674 g_SourceMod.BuildPath(Path_SM, path, sizeof(path), "translations/core.cfg");
676 if (g_LibSys.PathExists(path))
678 id = FindOrAddPhraseFile("core.cfg");
680 id = FindOrAddPhraseFile("core.phrases.txt");
683 g_pCorePhrases = GetFileByIndex(id);
686 bool Translator::GetLanguageByCode(const char *code, unsigned int *index)
690 if (!sm_trie_retrieve(m_pLCodeLookup, code, &_index))
697 *index = reinterpret_cast<unsigned int>(_index);
703 unsigned int Translator::GetLanguageCount()
705 return (unsigned int)m_Languages.size();
708 BaseStringTable *Translator::GetStringTable()
713 unsigned int Translator::FindOrAddPhraseFile(const char *phrase_file)
715 for (size_t i=0; i<m_Files.size(); i++)
717 if (strcmp(m_Files[i]->GetFilename(), phrase_file) == 0)
719 return (unsigned int)i;
723 CPhraseFile *pFile = new CPhraseFile(this, phrase_file);
724 unsigned int idx = (unsigned int)m_Files.size();
726 m_Files.push_back(pFile);
728 pFile->ReparseFile();
733 void Translator::RebuildLanguageDatabase(const char *lang_header_file)
735 /* Erase everything we have */
736 sm_trie_destroy(m_pLCodeLookup);
737 m_pLCodeLookup = sm_trie_create();
738 m_pStringTab->Reset();
740 for (size_t i=0; i<m_Languages.size(); i++)
742 delete m_Languages[i];
748 unsigned int line=0, col=0;
749 if ((err=g_TextParser.ParseFile_SMC(lang_header_file, this, &line, &col)) != SMCParse_Okay)
751 const char *str_err = g_TextParser.GetSMCErrorString(err);
754 str_err = m_CustomError.c_str();
757 g_Logger.LogError("[SM] Failed to parse language header file: \"%s\"", lang_header_file);
758 g_Logger.LogError("[SM] Parse error (line %d, column %d): %s", line, col, str_err);
763 if (!sm_trie_retrieve(m_pLCodeLookup, m_InitialLang, &serverLang))
765 g_Logger.LogError("Server language was set to bad language \"%s\" -- reverting to English", m_InitialLang);
767 strncopy(m_InitialLang, "en", sizeof(m_InitialLang));
768 m_ServerLang = LANGUAGE_ENGLISH;
771 m_ServerLang = reinterpret_cast<unsigned int>(serverLang);
773 if (!m_Languages.size())
775 g_Logger.LogError("[SM] Fatal error, no languages found! Translation will not work.");
778 for (size_t i=0; i<m_Files.size(); i++)
780 m_Files[i]->ReparseFile();
784 void Translator::ReadSMC_ParseStart()
786 m_InLanguageSection = false;
787 m_CustomError.clear();
790 SMCParseResult Translator::ReadSMC_NewSection(const char *name, bool opt_quotes)
792 if (!m_InLanguageSection)
794 if (strcmp(name, "Languages") == 0)
796 m_InLanguageSection = true;
800 if (!m_InLanguageSection)
802 g_Logger.LogError("[SM] Warning: Unrecognized section \"%s\" in languages.cfg", name);
805 return SMCParse_Continue;
808 SMCParseResult Translator::ReadSMC_LeavingSection()
810 m_InLanguageSection = false;
812 return SMCParse_Continue;
815 SMCParseResult Translator::ReadSMC_KeyValue(const char *key, const char *value, bool key_quotes, bool value_quotes)
817 size_t len = strlen(key);
821 g_Logger.LogError("[SM] Warning encountered parsing languages.cfg file.");
822 g_Logger.LogError("[SM] Invalid language code \"%s\" is being ignored.", key);
825 AddLanguage(key, value);
827 return SMCParse_Continue;
830 bool Translator::AddLanguage(const char *langcode, const char *description)
832 if (sm_trie_retrieve(m_pLCodeLookup, langcode, NULL))
837 Language *pLanguage = new Language;
838 unsigned int idx = m_Languages.size();
840 pLanguage->m_code2[0] = langcode[0];
841 pLanguage->m_code2[1] = langcode[1];
842 pLanguage->m_code2[2] = langcode[2];
843 pLanguage->m_FullName = m_pStringTab->AddString(description);
845 sm_trie_insert(m_pLCodeLookup, langcode, reinterpret_cast<void *>(idx));
847 m_Languages.push_back(pLanguage);
852 CPhraseFile *Translator::GetFileByIndex(unsigned int index)
854 if (index >= m_Files.size())
859 return m_Files[index];
862 size_t Translator::Translate(char *buffer, size_t maxlength, void **params, const Translation *pTrans)
864 void *new_params[MAX_TRANSLATE_PARAMS];
866 /* Rewrite the parameter order */
867 for (unsigned int i=0; i<pTrans->fmt_count; i++)
869 new_params[i] = params[pTrans->fmt_order[i]];
872 return gnprintf(buffer, maxlength, pTrans->szPhrase, new_params);
875 TransError Translator::CoreTrans(int client,
882 /* :TODO: do language stuff here */
886 return Trans_BadPhraseFile;
892 /* Using server lang temporarily until client lang stuff is implemented */
893 if ((err=g_pCorePhrases->GetTranslation(phrase, m_ServerLang, &trans)) != Trans_Okay)
898 size_t len = Translate(buffer, maxlength, params, &trans);
908 unsigned int Translator::GetServerLanguage()