1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/sourcepawn/compiler/sci18n.c Sun Mar 30 07:00:22 2008 +0000
1.3 @@ -0,0 +1,428 @@
1.4 +/* Codepage translation to Unicode, and UTF-8 support
1.5 + *
1.6 + * The translation is based on codepage mapping files that are distributed
1.7 + * by the Unicode consortium, see ftp://ftp.unicode.org/Public/MAPPINGS/.
1.8 + *
1.9 + * Character sets with a maximum of 256 codes are translated via a lookup
1.10 + * table (these are Single-Byte Character Sets). Character sets like Shift-JIS
1.11 + * with single-byte characters and multi-byte characters (introduced by a
1.12 + * leader byte) are split into two tables: the 256-entry lookup table for
1.13 + * the single-byte characters and an extended table for the multi-byte
1.14 + * characters. The extended table is allocated dynamically; the lookup table
1.15 + * is allocated statically, so loading SBCS tables cannot fail (if the tables
1.16 + * themselves are valid, of course).
1.17 + *
1.18 + * Copyright (c) ITB CompuPhase, 2004-2006
1.19 + *
1.20 + * This software is provided "as-is", without any express or implied warranty.
1.21 + * In no event will the authors be held liable for any damages arising from
1.22 + * the use of this software.
1.23 + *
1.24 + * Permission is granted to anyone to use this software for any purpose,
1.25 + * including commercial applications, and to alter it and redistribute it
1.26 + * freely, subject to the following restrictions:
1.27 + *
1.28 + * 1. The origin of this software must not be misrepresented; you must not
1.29 + * claim that you wrote the original software. If you use this software in
1.30 + * a product, an acknowledgment in the product documentation would be
1.31 + * appreciated but is not required.
1.32 + * 2. Altered source versions must be plainly marked as such, and must not be
1.33 + * misrepresented as being the original software.
1.34 + * 3. This notice may not be removed or altered from any source distribution.
1.35 + *
1.36 + * Version: $Id$
1.37 + */
1.38 +#include <assert.h>
1.39 +#include <stdio.h>
1.40 +#include <stddef.h>
1.41 +#include <stdlib.h>
1.42 +#include <string.h>
1.43 +#include "sc.h"
1.44 +
1.45 +#if !defined TRUE
1.46 + #define FALSE 0
1.47 + #define TRUE 1
1.48 +#endif
1.49 +#if !defined _MAX_PATH
1.50 + #define _MAX_PATH 250
1.51 +#endif
1.52 +#if !defined DIRSEP_CHAR
1.53 + #if defined LINUX || defined __FreeBSD__ || defined __OpenBSD__
1.54 + #define DIRSEP_CHAR '/'
1.55 + #elif defined macintosh
1.56 + #define DIRSEP_CHAR ':'
1.57 + #else
1.58 + #define DIRSEP_CHAR '\\'
1.59 + #endif
1.60 +#endif
1.61 +
1.62 +#if !defined ELEMENTS
1.63 + #define ELEMENTS(array) (sizeof(array) / sizeof(array[0]))
1.64 +#endif
1.65 +
1.66 +#if !defined NO_CODEPAGE
1.67 +
1.68 +#if !defined MAXCODEPAGE
1.69 + #define MAXCODEPAGE 12 /* typically "cp" + 4 digits + ".txt" */
1.70 +#endif
1.71 +#define INVALID 0xffffu /* 0xffff and 0xfffe are invalid Unicode characters */
1.72 +#define LEADBYTE 0xfffeu
1.73 +
1.74 +struct wordpair {
1.75 + unsigned short index;
1.76 + wchar_t code;
1.77 +};
1.78 +static char cprootpath[_MAX_PATH] = { DIRSEP_CHAR, '\0' };
1.79 +static wchar_t bytetable[256];
1.80 +static struct wordpair *wordtable = NULL;
1.81 +static unsigned wordtablesize = 0;
1.82 +static unsigned wordtabletop = 0;
1.83 +
1.84 +
1.85 +/* read in a line delimited by '\r' or '\n'; do NOT store the '\r' or '\n' into
1.86 + * the string and ignore empty lines
1.87 + * returns 1 for success and 0 for failure
1.88 + */
1.89 +static int cp_readline(FILE *fp,char *string,size_t size)
1.90 +{
1.91 + size_t count=0;
1.92 + int c;
1.93 + assert(size>1);
1.94 + while ((c=fgetc(fp))!=EOF && count<size-1) {
1.95 + if (c=='\r' || c=='\n') {
1.96 + if (count>0) /* '\r' or '\n' ends a string */
1.97 + break;
1.98 + /* if count==0, the line started with a '\r' or '\n', or perhaps line
1.99 + * ends in the file are '\r\n' and we read and stopped on the '\r' of
1.100 + * the preceding line
1.101 + */
1.102 + } else {
1.103 + string[count++]=(char)c;
1.104 + } /* if */
1.105 + } /* while */
1.106 + string[count]='\0';
1.107 + return count>0;
1.108 +}
1.109 +
1.110 +/* cp_path() sets the directory where all codepage files must be found (if
1.111 + * the parameter to cp_set() specifies a full path, that is used instead).
1.112 + * The path is specified into two parts: root and directory; the full path
1.113 + * for the codepage direcory is just the concatenation of the two, with a
1.114 + * directory separator in between. The directory is given in two parts,
1.115 + * because often a program already retrieves its "home" directory and the
1.116 + * codepages are most conveniently stored in a subdirectory of this home
1.117 + * directory.
1.118 + */
1.119 +SC_FUNC int cp_path(const char *root, const char *directory)
1.120 +{
1.121 + size_t len1,len2;
1.122 + int add_slash1,add_slash2;
1.123 +
1.124 + len1= (root!=NULL) ? strlen(root) : 0;
1.125 + add_slash1= (len1==0 || root[len1-1]!=DIRSEP_CHAR);
1.126 + len2= (directory!=NULL) ? strlen(directory) : 0;
1.127 + add_slash2= (len2>0 && root[len2-1]!=DIRSEP_CHAR);
1.128 + if (len1+add_slash1+len2+add_slash2>=(_MAX_PATH-MAXCODEPAGE))
1.129 + return FALSE; /* full filename may not fit */
1.130 + if (root!=NULL)
1.131 + strcpy(cprootpath,root);
1.132 + if (add_slash1) {
1.133 + assert(len1==0 || cprootpath[len1]=='\0');
1.134 + cprootpath[len1]=DIRSEP_CHAR;
1.135 + cprootpath[len1+1]='\0';
1.136 + } /* if */
1.137 + if (directory!=NULL)
1.138 + strcat(cprootpath,directory);
1.139 + if (add_slash2) {
1.140 + assert(cprootpath[len1+add_slash1+len2]=='\0');
1.141 + cprootpath[len1+add_slash1+len2]=DIRSEP_CHAR;
1.142 + cprootpath[len1+add_slash1+len2+1]='\0';
1.143 + } /* if */
1.144 + cp_set(NULL); /* start with a "linear" table (no translation) */
1.145 + return TRUE;
1.146 +}
1.147 +
1.148 +/* cp_set() loads a codepage from a file. The name parameter may be a
1.149 + * filename (including a full path) or it may be a partial codepage name.
1.150 + * If the name parameter is NULL, the codepage is cleared to be a "linear"
1.151 + * table (no translation).
1.152 + * The following files are attempted to open (where <name> specifies the
1.153 + * value of the parameter):
1.154 + * <name>
1.155 + * <cprootpath>/<name>
1.156 + * <cprootpath>/<name>.txt
1.157 + * <cprootpath>/cp<name>
1.158 + * <cprootpath>/cp<name>.txt
1.159 + */
1.160 +SC_FUNC int cp_set(const char *name)
1.161 +{
1.162 + char filename[_MAX_PATH];
1.163 + FILE *fp=NULL;
1.164 + unsigned index;
1.165 +
1.166 + /* for name==NULL, set up an identity table */
1.167 + if (name==NULL || *name=='\0') {
1.168 + if (wordtable!=NULL) {
1.169 + free(wordtable);
1.170 + wordtable=NULL;
1.171 + wordtablesize=0;
1.172 + wordtabletop=0;
1.173 + } /* if */
1.174 + for (index=0; index<ELEMENTS(bytetable); index++)
1.175 + bytetable[index]=(wchar_t)index;
1.176 + return TRUE;
1.177 + } /* if */
1.178 +
1.179 + /* try to open the file as-is */
1.180 + if (strchr(name,DIRSEP_CHAR)!=NULL)
1.181 + fp=fopen(name,"rt");
1.182 + if (fp==NULL) {
1.183 + /* try opening the file in the "root path" for codepages */
1.184 + if (strlen(name)>MAXCODEPAGE)
1.185 + return 0;
1.186 + assert(strlen(name)+strlen(cprootpath)<_MAX_PATH);
1.187 + strcpy(filename,cprootpath);
1.188 + strcat(filename,name);
1.189 + fp=fopen(filename,"rt");
1.190 + } /* if */
1.191 + if (fp==NULL) {
1.192 + /* try opening the file in the "root path" for codepages, with a ".txt" extension */
1.193 + if (strlen(name)+4>=MAXCODEPAGE)
1.194 + return 0;
1.195 + assert(strlen(filename)+4<_MAX_PATH);
1.196 + strcat(filename,".txt");
1.197 + fp=fopen(filename,"rt");
1.198 + } /* if */
1.199 + if (fp==NULL) {
1.200 + /* try opening the file in the "root path" for codepages, with "cp" prefixed before the name */
1.201 + if (strlen(name)+2>MAXCODEPAGE)
1.202 + return 0;
1.203 + assert(2+strlen(name)+strlen(cprootpath)<_MAX_PATH);
1.204 + strcpy(filename,cprootpath);
1.205 + strcat(filename,"cp");
1.206 + strcat(filename,name);
1.207 + fp=fopen(filename,"rt");
1.208 + } /* if */
1.209 + if (fp==NULL) {
1.210 + /* try opening the file in the "root path" for codepages, with "cp" prefixed an ".txt" appended */
1.211 + if (strlen(name)+2+4>MAXCODEPAGE)
1.212 + return 0;
1.213 + assert(strlen(filename)+4<_MAX_PATH);
1.214 + strcat(filename,".txt");
1.215 + fp=fopen(filename,"rt");
1.216 + } /* if */
1.217 + if (fp==NULL)
1.218 + return FALSE; /* all failed */
1.219 +
1.220 + /* clear the tables */
1.221 + for (index=0; index<ELEMENTS(bytetable); index++)
1.222 + bytetable[index]=INVALID; /* special code meaning "not found" */
1.223 + assert(wordtablesize==0 && wordtabletop==0 && wordtable==NULL
1.224 + || wordtablesize>0 && wordtable!=NULL);
1.225 + if (wordtable!=NULL) {
1.226 + free(wordtable);
1.227 + wordtable=NULL;
1.228 + wordtablesize=0;
1.229 + wordtabletop=0;
1.230 + } /* if */
1.231 +
1.232 + /* read in the table */
1.233 + while (cp_readline(fp,filename,sizeof filename)) {
1.234 + char *ptr;
1.235 + if ((ptr=strchr(filename,'#'))!=NULL)
1.236 + *ptr='\0'; /* strip of comment */
1.237 + for (ptr=filename; *ptr>0 && *ptr<' '; ptr++)
1.238 + /* nothing */; /* skip leading whitespace */
1.239 + if (*ptr!='\0') {
1.240 + /* content on line */
1.241 + unsigned code=LEADBYTE;
1.242 + int num=sscanf(ptr,"%i %i",&index,&code);
1.243 + /* if sscanf() returns 1 and the index is in range 0..255, then the
1.244 + * code is a DBCS lead byte; if sscanf() returns 2 and index>=256, this
1.245 + * is a double byte pair (lead byte + follower)
1.246 + */
1.247 + if (num>=1 && index<256) {
1.248 + bytetable[index]=(wchar_t)code;
1.249 + } else if (num==2 && index>=256 && index<LEADBYTE) {
1.250 + /* store the DBCS character in wordtable */
1.251 + if (wordtabletop>=wordtablesize) {
1.252 + /* grow the list */
1.253 + int newsize;
1.254 + struct wordpair *newblock;
1.255 + newsize= (wordtablesize==0) ? 128 : 2*wordtablesize;
1.256 + newblock=(struct wordpair *)malloc(newsize*sizeof(*wordtable));
1.257 + if (newblock!=NULL) {
1.258 + memcpy(newblock,wordtable,wordtabletop*sizeof(*wordtable));
1.259 + free(wordtable);
1.260 + wordtable=newblock;
1.261 + wordtablesize=newsize;
1.262 + } /* if */
1.263 + } /* if */
1.264 + if (wordtabletop<wordtablesize) {
1.265 + /* insert at sorted position */
1.266 + int pos=wordtabletop;
1.267 + assert(wordtable!=NULL);
1.268 + while (pos>0 && (unsigned)wordtable[pos-1].index>index) {
1.269 + wordtable[pos]=wordtable[pos-1];
1.270 + pos--;
1.271 + } /* while */
1.272 + wordtable[pos].index=(unsigned short)index;
1.273 + wordtable[pos].code=(wchar_t)code;
1.274 + } /* if */
1.275 + } /* if */
1.276 + } /* if */
1.277 + } /* while */
1.278 +
1.279 + fclose(fp);
1.280 + return TRUE;
1.281 +}
1.282 +
1.283 +SC_FUNC cell cp_translate(const unsigned char *string,const unsigned char **endptr)
1.284 +{
1.285 + wchar_t result;
1.286 +
1.287 + result=bytetable[*string++];
1.288 + /* check whether this is a leader code */
1.289 + if ((unsigned)result==LEADBYTE && wordtable!=NULL) {
1.290 + /* look up the code via binary search */
1.291 + int low,high,mid;
1.292 + unsigned short index=(unsigned short)(((*(string-1)) << 8) | *string);
1.293 + string++;
1.294 + assert(wordtabletop>0);
1.295 + low=0;
1.296 + high=wordtabletop-1;
1.297 + while (low<high) {
1.298 + mid=(low+high)/2;
1.299 + assert(low<=mid && mid<high);
1.300 + if (index>wordtable[mid].index)
1.301 + low=mid+1;
1.302 + else
1.303 + high=mid;
1.304 + } /* while */
1.305 + assert(low==high);
1.306 + if (wordtable[low].index==index)
1.307 + result=wordtable[low].code;
1.308 + } /* if */
1.309 +
1.310 + if (endptr!=NULL)
1.311 + *endptr=string;
1.312 + return (cell)result;
1.313 +}
1.314 +
1.315 +#endif /* NO_CODEPAGE */
1.316 +
1.317 +#if !defined NO_UTF8
1.318 +SC_FUNC cell get_utf8_char(const unsigned char *string,const unsigned char **endptr)
1.319 +{
1.320 + int follow=0;
1.321 + long lowmark=0;
1.322 + unsigned char ch;
1.323 + cell result=0;
1.324 +
1.325 + if (endptr!=NULL)
1.326 + *endptr=string;
1.327 +
1.328 + for ( ;; ) {
1.329 + ch=*string++;
1.330 +
1.331 + if (follow>0 && (ch & 0xc0)==0x80) {
1.332 + /* leader code is active, combine with earlier code */
1.333 + result=(result << 6) | (ch & 0x3f);
1.334 + if (--follow==0) {
1.335 + /* encoding a character in more bytes than is strictly needed,
1.336 + * is not really valid UTF-8; we are strict here to increase
1.337 + * the chance of heuristic dectection of non-UTF-8 text
1.338 + * (JAVA writes zero bytes as a 2-byte code UTF-8, which is invalid)
1.339 + */
1.340 + if (result<lowmark)
1.341 + return -1;
1.342 + /* the code positions 0xd800--0xdfff and 0xfffe & 0xffff do not
1.343 + * exist in UCS-4 (and hence, they do not exist in Unicode)
1.344 + */
1.345 + if (result>=0xd800 && result<=0xdfff || result==0xfffe || result==0xffff)
1.346 + return -1;
1.347 + } /* if */
1.348 + break;
1.349 + } else if (follow==0 && (ch & 0x80)==0x80) {
1.350 + /* UTF-8 leader code */
1.351 + if ((ch & 0xe0)==0xc0) {
1.352 + /* 110xxxxx 10xxxxxx */
1.353 + follow=1;
1.354 + lowmark=0x80L;
1.355 + result=ch & 0x1f;
1.356 + } else if ((ch & 0xf0)==0xe0) {
1.357 + /* 1110xxxx 10xxxxxx 10xxxxxx (16 bits, BMP plane) */
1.358 + follow=2;
1.359 + lowmark=0x800L;
1.360 + result=ch & 0x0f;
1.361 + } else if ((ch & 0xf8)==0xf0) {
1.362 + /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
1.363 + follow=3;
1.364 + lowmark=0x10000L;
1.365 + result=ch & 0x07;
1.366 + } else if ((ch & 0xfc)==0xf8) {
1.367 + /* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
1.368 + follow=4;
1.369 + lowmark=0x200000L;
1.370 + result=ch & 0x03;
1.371 + } else if ((ch & 0xfe)==0xfc) {
1.372 + /* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx (32 bits) */
1.373 + follow=5;
1.374 + lowmark=0x4000000L;
1.375 + result=ch & 0x01;
1.376 + } else {
1.377 + /* this is invalid UTF-8 */
1.378 + return -1;
1.379 + } /* if */
1.380 + } else if (follow==0 && (ch & 0x80)==0x00) {
1.381 + /* 0xxxxxxx (US-ASCII) */
1.382 + result=ch;
1.383 + break;
1.384 + } else {
1.385 + /* this is invalid UTF-8 */
1.386 + return -1;
1.387 + } /* if */
1.388 +
1.389 + } /* for */
1.390 +
1.391 + if (endptr!=NULL)
1.392 + *endptr=string;
1.393 + return result;
1.394 +}
1.395 +#endif
1.396 +
1.397 +SC_FUNC int scan_utf8(FILE *fp,const char *filename)
1.398 +{
1.399 + #if defined NO_UTF8
1.400 + return 0;
1.401 + #else
1.402 + void *resetpos=pc_getpossrc(fp);
1.403 + int utf8=TRUE;
1.404 + int firstchar=TRUE,bom_found=FALSE;
1.405 + const unsigned char *ptr;
1.406 +
1.407 + while (utf8 && pc_readsrc(fp,pline,sLINEMAX)!=NULL) {
1.408 + ptr=pline;
1.409 + if (firstchar) {
1.410 + /* check whether the very first character on the very first line
1.411 + * starts with a BYTE order mark
1.412 + */
1.413 + cell c=get_utf8_char(ptr,&ptr);
1.414 + bom_found= (c==0xfeff);
1.415 + utf8= (c>=0);
1.416 + firstchar=FALSE;
1.417 + } /* if */
1.418 + while (utf8 && *ptr!='\0')
1.419 + utf8= (get_utf8_char(ptr,&ptr)>=0);
1.420 + } /* while */
1.421 + pc_resetsrc(fp,resetpos);
1.422 + if (bom_found) {
1.423 + unsigned char bom[3];
1.424 + if (!utf8)
1.425 + error(77,filename); /* malformed UTF-8 encoding */
1.426 + pc_readsrc(fp,bom,3);
1.427 + assert(bom[0]==0xef && bom[1]==0xbb && bom[2]==0xbf);
1.428 + } /* if */
1.429 + return utf8;
1.430 + #endif /* NO_UTF8 */
1.431 +}
