1.1 --- a/compiler-init/sci18n.c Sat Jul 15 01:36:48 2006 +0000
1.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000
1.3 @@ -1,428 +0,0 @@
1.4 -/* Codepage translation to Unicode, and UTF-8 support
1.5 - *
1.6 - * The translation is based on codepage mapping files that are distributed
1.7 - * by the Unicode consortium, see ftp://ftp.unicode.org/Public/MAPPINGS/.
1.8 - *
1.9 - * Character sets with a maximum of 256 codes are translated via a lookup
1.10 - * table (these are Single-Byte Character Sets). Character sets like Shift-JIS
1.11 - * with single-byte characters and multi-byte characters (introduced by a
1.12 - * leader byte) are split into two tables: the 256-entry lookup table for
1.13 - * the single-byte characters and an extended table for the multi-byte
1.14 - * characters. The extended table is allocated dynamically; the lookup table
1.15 - * is allocated statically, so loading SBCS tables cannot fail (if the tables
1.16 - * themselves are valid, of course).
1.17 - *
1.18 - * Copyright (c) ITB CompuPhase, 2004-2006
1.19 - *
1.20 - * This software is provided "as-is", without any express or implied warranty.
1.21 - * In no event will the authors be held liable for any damages arising from
1.22 - * the use of this software.
1.23 - *
1.24 - * Permission is granted to anyone to use this software for any purpose,
1.25 - * including commercial applications, and to alter it and redistribute it
1.26 - * freely, subject to the following restrictions:
1.27 - *
1.28 - * 1. The origin of this software must not be misrepresented; you must not
1.29 - * claim that you wrote the original software. If you use this software in
1.30 - * a product, an acknowledgment in the product documentation would be
1.31 - * appreciated but is not required.
1.32 - * 2. Altered source versions must be plainly marked as such, and must not be
1.33 - * misrepresented as being the original software.
1.34 - * 3. This notice may not be removed or altered from any source distribution.
1.35 - *
1.36 - * Version: $Id: CODEPAGE.C,v 1.0 2004-02-18 12:13:04+01 thiadmer Exp thiadmer $
1.37 - */
1.38 -#include <assert.h>
1.39 -#include <stdio.h>
1.40 -#include <stddef.h>
1.41 -#include <stdlib.h>
1.42 -#include <string.h>
1.43 -#include "sc.h"
1.44 -
1.45 -#if !defined TRUE
1.46 - #define FALSE 0
1.47 - #define TRUE 1
1.48 -#endif
1.49 -#if !defined _MAX_PATH
1.50 - #define _MAX_PATH 250
1.51 -#endif
1.52 -#if !defined DIRSEP_CHAR
1.53 - #if defined LINUX || defined __FreeBSD__ || defined __OpenBSD__
1.54 - #define DIRSEP_CHAR '/'
1.55 - #elif defined macintosh
1.56 - #define DIRSEP_CHAR ':'
1.57 - #else
1.58 - #define DIRSEP_CHAR '\\'
1.59 - #endif
1.60 -#endif
1.61 -
1.62 -#if !defined ELEMENTS
1.63 - #define ELEMENTS(array) (sizeof(array) / sizeof(array[0]))
1.64 -#endif
1.65 -
1.66 -#if !defined NO_CODEPAGE
1.67 -
1.68 -#if !defined MAXCODEPAGE
1.69 - #define MAXCODEPAGE 12 /* typically "cp" + 4 digits + ".txt" */
1.70 -#endif
1.71 -#define INVALID 0xffffu /* 0xffff and 0xfffe are invalid Unicode characters */
1.72 -#define LEADBYTE 0xfffeu
1.73 -
1.74 -struct wordpair {
1.75 - unsigned short index;
1.76 - wchar_t code;
1.77 -};
1.78 -static char cprootpath[_MAX_PATH] = { DIRSEP_CHAR, '\0' };
1.79 -static wchar_t bytetable[256];
1.80 -static struct wordpair *wordtable = NULL;
1.81 -static unsigned wordtablesize = 0;
1.82 -static unsigned wordtabletop = 0;
1.83 -
1.84 -
1.85 -/* read in a line delimited by '\r' or '\n'; do NOT store the '\r' or '\n' into
1.86 - * the string and ignore empty lines
1.87 - * returns 1 for success and 0 for failure
1.88 - */
1.89 -static int cp_readline(FILE *fp,char *string,size_t size)
1.90 -{
1.91 - size_t count=0;
1.92 - int c;
1.93 - assert(size>1);
1.94 - while ((c=fgetc(fp))!=EOF && count<size-1) {
1.95 - if (c=='\r' || c=='\n') {
1.96 - if (count>0) /* '\r' or '\n' ends a string */
1.97 - break;
1.98 - /* if count==0, the line started with a '\r' or '\n', or perhaps line
1.99 - * ends in the file are '\r\n' and we read and stopped on the '\r' of
1.100 - * the preceding line
1.101 - */
1.102 - } else {
1.103 - string[count++]=(char)c;
1.104 - } /* if */
1.105 - } /* while */
1.106 - string[count]='\0';
1.107 - return count>0;
1.108 -}
1.109 -
1.110 -/* cp_path() sets the directory where all codepage files must be found (if
1.111 - * the parameter to cp_set() specifies a full path, that is used instead).
1.112 - * The path is specified into two parts: root and directory; the full path
1.113 - * for the codepage direcory is just the concatenation of the two, with a
1.114 - * directory separator in between. The directory is given in two parts,
1.115 - * because often a program already retrieves its "home" directory and the
1.116 - * codepages are most conveniently stored in a subdirectory of this home
1.117 - * directory.
1.118 - */
1.119 -SC_FUNC int cp_path(const char *root, const char *directory)
1.120 -{
1.121 - size_t len1,len2;
1.122 - int add_slash1,add_slash2;
1.123 -
1.124 - len1= (root!=NULL) ? strlen(root) : 0;
1.125 - add_slash1= (len1==0 || root[len1-1]!=DIRSEP_CHAR);
1.126 - len2= (directory!=NULL) ? strlen(directory) : 0;
1.127 - add_slash2= (len2>0 && root[len2-1]!=DIRSEP_CHAR);
1.128 - if (len1+add_slash1+len2+add_slash2>=(_MAX_PATH-MAXCODEPAGE))
1.129 - return FALSE; /* full filename may not fit */
1.130 - if (root!=NULL)
1.131 - strcpy(cprootpath,root);
1.132 - if (add_slash1) {
1.133 - assert(len1==0 || cprootpath[len1]=='\0');
1.134 - cprootpath[len1]=DIRSEP_CHAR;
1.135 - cprootpath[len1+1]='\0';
1.136 - } /* if */
1.137 - if (directory!=NULL)
1.138 - strcat(cprootpath,directory);
1.139 - if (add_slash2) {
1.140 - assert(cprootpath[len1+add_slash1+len2]=='\0');
1.141 - cprootpath[len1+add_slash1+len2]=DIRSEP_CHAR;
1.142 - cprootpath[len1+add_slash1+len2+1]='\0';
1.143 - } /* if */
1.144 - cp_set(NULL); /* start with a "linear" table (no translation) */
1.145 - return TRUE;
1.146 -}
1.147 -
1.148 -/* cp_set() loads a codepage from a file. The name parameter may be a
1.149 - * filename (including a full path) or it may be a partial codepage name.
1.150 - * If the name parameter is NULL, the codepage is cleared to be a "linear"
1.151 - * table (no translation).
1.152 - * The following files are attempted to open (where <name> specifies the
1.153 - * value of the parameter):
1.154 - * <name>
1.155 - * <cprootpath>/<name>
1.156 - * <cprootpath>/<name>.txt
1.157 - * <cprootpath>/cp<name>
1.158 - * <cprootpath>/cp<name>.txt
1.159 - */
1.160 -SC_FUNC int cp_set(const char *name)
1.161 -{
1.162 - char filename[_MAX_PATH];
1.163 - FILE *fp=NULL;
1.164 - unsigned index;
1.165 -
1.166 - /* for name==NULL, set up an identity table */
1.167 - if (name==NULL || *name=='\0') {
1.168 - if (wordtable!=NULL) {
1.169 - free(wordtable);
1.170 - wordtable=NULL;
1.171 - wordtablesize=0;
1.172 - wordtabletop=0;
1.173 - } /* if */
1.174 - for (index=0; index<ELEMENTS(bytetable); index++)
1.175 - bytetable[index]=(wchar_t)index;
1.176 - return TRUE;
1.177 - } /* if */
1.178 -
1.179 - /* try to open the file as-is */
1.180 - if (strchr(name,DIRSEP_CHAR)!=NULL)
1.181 - fp=fopen(name,"rt");
1.182 - if (fp==NULL) {
1.183 - /* try opening the file in the "root path" for codepages */
1.184 - if (strlen(name)>MAXCODEPAGE)
1.185 - return 0;
1.186 - assert(strlen(name)+strlen(cprootpath)<_MAX_PATH);
1.187 - strcpy(filename,cprootpath);
1.188 - strcat(filename,name);
1.189 - fp=fopen(filename,"rt");
1.190 - } /* if */
1.191 - if (fp==NULL) {
1.192 - /* try opening the file in the "root path" for codepages, with a ".txt" extension */
1.193 - if (strlen(name)+4>=MAXCODEPAGE)
1.194 - return 0;
1.195 - assert(strlen(filename)+4<_MAX_PATH);
1.196 - strcat(filename,".txt");
1.197 - fp=fopen(filename,"rt");
1.198 - } /* if */
1.199 - if (fp==NULL) {
1.200 - /* try opening the file in the "root path" for codepages, with "cp" prefixed before the name */
1.201 - if (strlen(name)+2>MAXCODEPAGE)
1.202 - return 0;
1.203 - assert(2+strlen(name)+strlen(cprootpath)<_MAX_PATH);
1.204 - strcpy(filename,cprootpath);
1.205 - strcat(filename,"cp");
1.206 - strcat(filename,name);
1.207 - fp=fopen(filename,"rt");
1.208 - } /* if */
1.209 - if (fp==NULL) {
1.210 - /* try opening the file in the "root path" for codepages, with "cp" prefixed an ".txt" appended */
1.211 - if (strlen(name)+2+4>MAXCODEPAGE)
1.212 - return 0;
1.213 - assert(strlen(filename)+4<_MAX_PATH);
1.214 - strcat(filename,".txt");
1.215 - fp=fopen(filename,"rt");
1.216 - } /* if */
1.217 - if (fp==NULL)
1.218 - return FALSE; /* all failed */
1.219 -
1.220 - /* clear the tables */
1.221 - for (index=0; index<ELEMENTS(bytetable); index++)
1.222 - bytetable[index]=INVALID; /* special code meaning "not found" */
1.223 - assert(wordtablesize==0 && wordtabletop==0 && wordtable==NULL
1.224 - || wordtablesize>0 && wordtable!=NULL);
1.225 - if (wordtable!=NULL) {
1.226 - free(wordtable);
1.227 - wordtable=NULL;
1.228 - wordtablesize=0;
1.229 - wordtabletop=0;
1.230 - } /* if */
1.231 -
1.232 - /* read in the table */
1.233 - while (cp_readline(fp,filename,sizeof filename)) {
1.234 - char *ptr;
1.235 - if ((ptr=strchr(filename,'#'))!=NULL)
1.236 - *ptr='\0'; /* strip of comment */
1.237 - for (ptr=filename; *ptr>0 && *ptr<' '; ptr++)
1.238 - /* nothing */; /* skip leading whitespace */
1.239 - if (*ptr!='\0') {
1.240 - /* content on line */
1.241 - unsigned code=LEADBYTE;
1.242 - int num=sscanf(ptr,"%i %i",&index,&code);
1.243 - /* if sscanf() returns 1 and the index is in range 0..255, then the
1.244 - * code is a DBCS lead byte; if sscanf() returns 2 and index>=256, this
1.245 - * is a double byte pair (lead byte + follower)
1.246 - */
1.247 - if (num>=1 && index<256) {
1.248 - bytetable[index]=(wchar_t)code;
1.249 - } else if (num==2 && index>=256 && index<LEADBYTE) {
1.250 - /* store the DBCS character in wordtable */
1.251 - if (wordtabletop>=wordtablesize) {
1.252 - /* grow the list */
1.253 - int newsize;
1.254 - struct wordpair *newblock;
1.255 - newsize= (wordtablesize==0) ? 128 : 2*wordtablesize;
1.256 - newblock=(struct wordpair *)malloc(newsize*sizeof(*wordtable));
1.257 - if (newblock!=NULL) {
1.258 - memcpy(newblock,wordtable,wordtabletop*sizeof(*wordtable));
1.259 - free(wordtable);
1.260 - wordtable=newblock;
1.261 - wordtablesize=newsize;
1.262 - } /* if */
1.263 - } /* if */
1.264 - if (wordtabletop<wordtablesize) {
1.265 - /* insert at sorted position */
1.266 - int pos=wordtabletop;
1.267 - assert(wordtable!=NULL);
1.268 - while (pos>0 && (unsigned)wordtable[pos-1].index>index) {
1.269 - wordtable[pos]=wordtable[pos-1];
1.270 - pos--;
1.271 - } /* while */
1.272 - wordtable[pos].index=(unsigned short)index;
1.273 - wordtable[pos].code=(wchar_t)code;
1.274 - } /* if */
1.275 - } /* if */
1.276 - } /* if */
1.277 - } /* while */
1.278 -
1.279 - fclose(fp);
1.280 - return TRUE;
1.281 -}
1.282 -
1.283 -SC_FUNC cell cp_translate(const unsigned char *string,const unsigned char **endptr)
1.284 -{
1.285 - wchar_t result;
1.286 -
1.287 - result=bytetable[*string++];
1.288 - /* check whether this is a leader code */
1.289 - if ((unsigned)result==LEADBYTE && wordtable!=NULL) {
1.290 - /* look up the code via binary search */
1.291 - int low,high,mid;
1.292 - unsigned short index=(unsigned short)(((*(string-1)) << 8) | *string);
1.293 - string++;
1.294 - assert(wordtabletop>0);
1.295 - low=0;
1.296 - high=wordtabletop-1;
1.297 - while (low<high) {
1.298 - mid=(low+high)/2;
1.299 - assert(low<=mid && mid<high);
1.300 - if (index>wordtable[mid].index)
1.301 - low=mid+1;
1.302 - else
1.303 - high=mid;
1.304 - } /* while */
1.305 - assert(low==high);
1.306 - if (wordtable[low].index==index)
1.307 - result=wordtable[low].code;
1.308 - } /* if */
1.309 -
1.310 - if (endptr!=NULL)
1.311 - *endptr=string;
1.312 - return (cell)result;
1.313 -}
1.314 -
1.315 -#endif /* NO_CODEPAGE */
1.316 -
1.317 -#if !defined NO_UTF8
1.318 -SC_FUNC cell get_utf8_char(const unsigned char *string,const unsigned char **endptr)
1.319 -{
1.320 - int follow=0;
1.321 - long lowmark=0;
1.322 - unsigned char ch;
1.323 - cell result=0;
1.324 -
1.325 - if (endptr!=NULL)
1.326 - *endptr=string;
1.327 -
1.328 - for ( ;; ) {
1.329 - ch=*string++;
1.330 -
1.331 - if (follow>0 && (ch & 0xc0)==0x80) {
1.332 - /* leader code is active, combine with earlier code */
1.333 - result=(result << 6) | (ch & 0x3f);
1.334 - if (--follow==0) {
1.335 - /* encoding a character in more bytes than is strictly needed,
1.336 - * is not really valid UTF-8; we are strict here to increase
1.337 - * the chance of heuristic dectection of non-UTF-8 text
1.338 - * (JAVA writes zero bytes as a 2-byte code UTF-8, which is invalid)
1.339 - */
1.340 - if (result<lowmark)
1.341 - return -1;
1.342 - /* the code positions 0xd800--0xdfff and 0xfffe & 0xffff do not
1.343 - * exist in UCS-4 (and hence, they do not exist in Unicode)
1.344 - */
1.345 - if (result>=0xd800 && result<=0xdfff || result==0xfffe || result==0xffff)
1.346 - return -1;
1.347 - } /* if */
1.348 - break;
1.349 - } else if (follow==0 && (ch & 0x80)==0x80) {
1.350 - /* UTF-8 leader code */
1.351 - if ((ch & 0xe0)==0xc0) {
1.352 - /* 110xxxxx 10xxxxxx */
1.353 - follow=1;
1.354 - lowmark=0x80L;
1.355 - result=ch & 0x1f;
1.356 - } else if ((ch & 0xf0)==0xe0) {
1.357 - /* 1110xxxx 10xxxxxx 10xxxxxx (16 bits, BMP plane) */
1.358 - follow=2;
1.359 - lowmark=0x800L;
1.360 - result=ch & 0x0f;
1.361 - } else if ((ch & 0xf8)==0xf0) {
1.362 - /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
1.363 - follow=3;
1.364 - lowmark=0x10000L;
1.365 - result=ch & 0x07;
1.366 - } else if ((ch & 0xfc)==0xf8) {
1.367 - /* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
1.368 - follow=4;
1.369 - lowmark=0x200000L;
1.370 - result=ch & 0x03;
1.371 - } else if ((ch & 0xfe)==0xfc) {
1.372 - /* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx (32 bits) */
1.373 - follow=5;
1.374 - lowmark=0x4000000L;
1.375 - result=ch & 0x01;
1.376 - } else {
1.377 - /* this is invalid UTF-8 */
1.378 - return -1;
1.379 - } /* if */
1.380 - } else if (follow==0 && (ch & 0x80)==0x00) {
1.381 - /* 0xxxxxxx (US-ASCII) */
1.382 - result=ch;
1.383 - break;
1.384 - } else {
1.385 - /* this is invalid UTF-8 */
1.386 - return -1;
1.387 - } /* if */
1.388 -
1.389 - } /* for */
1.390 -
1.391 - if (endptr!=NULL)
1.392 - *endptr=string;
1.393 - return result;
1.394 -}
1.395 -#endif
1.396 -
1.397 -SC_FUNC int scan_utf8(FILE *fp,const char *filename)
1.398 -{
1.399 - #if defined NO_UTF8
1.400 - return 0;
1.401 - #else
1.402 - void *resetpos=pc_getpossrc(fp);
1.403 - int utf8=TRUE;
1.404 - int firstchar=TRUE,bom_found=FALSE;
1.405 - const unsigned char *ptr;
1.406 -
1.407 - while (utf8 && pc_readsrc(fp,pline,sLINEMAX)!=NULL) {
1.408 - ptr=pline;
1.409 - if (firstchar) {
1.410 - /* check whether the very first character on the very first line
1.411 - * starts with a BYTE order mark
1.412 - */
1.413 - cell c=get_utf8_char(ptr,&ptr);
1.414 - bom_found= (c==0xfeff);
1.415 - utf8= (c>=0);
1.416 - firstchar=FALSE;
1.417 - } /* if */
1.418 - while (utf8 && *ptr!='\0')
1.419 - utf8= (get_utf8_char(ptr,&ptr)>=0);
1.420 - } /* while */
1.421 - pc_resetsrc(fp,resetpos);
1.422 - if (bom_found) {
1.423 - unsigned char bom[3];
1.424 - if (!utf8)
1.425 - error(77,filename); /* malformed UTF-8 encoding */
1.426 - pc_readsrc(fp,bom,3);
1.427 - assert(bom[0]==0xef && bom[1]==0xbb && bom[2]==0xbf);
1.428 - } /* if */
1.429 - return utf8;
1.430 - #endif /* NO_UTF8 */
1.431 -}
