dvander@27: /* Codepage translation to Unicode, and UTF-8 support dvander@27: * dvander@27: * The translation is based on codepage mapping files that are distributed dvander@27: * by the Unicode consortium, see ftp://ftp.unicode.org/Public/MAPPINGS/. dvander@27: * dvander@27: * Character sets with a maximum of 256 codes are translated via a lookup dvander@27: * table (these are Single-Byte Character Sets). Character sets like Shift-JIS dvander@27: * with single-byte characters and multi-byte characters (introduced by a dvander@27: * leader byte) are split into two tables: the 256-entry lookup table for dvander@27: * the single-byte characters and an extended table for the multi-byte dvander@27: * characters. The extended table is allocated dynamically; the lookup table dvander@27: * is allocated statically, so loading SBCS tables cannot fail (if the tables dvander@27: * themselves are valid, of course). dvander@27: * dvander@27: * Copyright (c) ITB CompuPhase, 2004-2006 dvander@27: * dvander@27: * This software is provided "as-is", without any express or implied warranty. dvander@27: * In no event will the authors be held liable for any damages arising from dvander@27: * the use of this software. dvander@27: * dvander@27: * Permission is granted to anyone to use this software for any purpose, dvander@27: * including commercial applications, and to alter it and redistribute it dvander@27: * freely, subject to the following restrictions: dvander@27: * dvander@27: * 1. The origin of this software must not be misrepresented; you must not dvander@27: * claim that you wrote the original software. If you use this software in dvander@27: * a product, an acknowledgment in the product documentation would be dvander@27: * appreciated but is not required. dvander@27: * 2. Altered source versions must be plainly marked as such, and must not be dvander@27: * misrepresented as being the original software. dvander@27: * 3. This notice may not be removed or altered from any source distribution. dvander@27: * faluco@62: * Version: $Id: sci18n.c 3612 2006-07-22 09:59:46Z thiadmer $ dvander@27: */ dvander@27: #include dvander@27: #include dvander@27: #include dvander@27: #include dvander@27: #include dvander@27: #include "sc.h" dvander@27: dvander@27: #if !defined TRUE dvander@27: #define FALSE 0 dvander@27: #define TRUE 1 dvander@27: #endif dvander@27: #if !defined _MAX_PATH dvander@27: #define _MAX_PATH 250 dvander@27: #endif dvander@27: #if !defined DIRSEP_CHAR dvander@27: #if defined LINUX || defined __FreeBSD__ || defined __OpenBSD__ dvander@27: #define DIRSEP_CHAR '/' dvander@27: #elif defined macintosh dvander@27: #define DIRSEP_CHAR ':' dvander@27: #else dvander@27: #define DIRSEP_CHAR '\\' dvander@27: #endif dvander@27: #endif dvander@27: dvander@27: #if !defined ELEMENTS dvander@27: #define ELEMENTS(array) (sizeof(array) / sizeof(array[0])) dvander@27: #endif dvander@27: dvander@27: #if !defined NO_CODEPAGE dvander@27: dvander@27: #if !defined MAXCODEPAGE dvander@27: #define MAXCODEPAGE 12 /* typically "cp" + 4 digits + ".txt" */ dvander@27: #endif dvander@27: #define INVALID 0xffffu /* 0xffff and 0xfffe are invalid Unicode characters */ dvander@27: #define LEADBYTE 0xfffeu dvander@27: dvander@27: struct wordpair { dvander@27: unsigned short index; dvander@27: wchar_t code; dvander@27: }; dvander@27: static char cprootpath[_MAX_PATH] = { DIRSEP_CHAR, '\0' }; dvander@27: static wchar_t bytetable[256]; dvander@27: static struct wordpair *wordtable = NULL; dvander@27: static unsigned wordtablesize = 0; dvander@27: static unsigned wordtabletop = 0; dvander@27: dvander@27: dvander@27: /* read in a line delimited by '\r' or '\n'; do NOT store the '\r' or '\n' into dvander@27: * the string and ignore empty lines dvander@27: * returns 1 for success and 0 for failure dvander@27: */ dvander@27: static int cp_readline(FILE *fp,char *string,size_t size) dvander@27: { dvander@27: size_t count=0; dvander@27: int c; dvander@27: assert(size>1); dvander@27: while ((c=fgetc(fp))!=EOF && count0) /* '\r' or '\n' ends a string */ dvander@27: break; dvander@27: /* if count==0, the line started with a '\r' or '\n', or perhaps line dvander@27: * ends in the file are '\r\n' and we read and stopped on the '\r' of dvander@27: * the preceding line dvander@27: */ dvander@27: } else { dvander@27: string[count++]=(char)c; dvander@27: } /* if */ dvander@27: } /* while */ dvander@27: string[count]='\0'; dvander@27: return count>0; dvander@27: } dvander@27: dvander@27: /* cp_path() sets the directory where all codepage files must be found (if dvander@27: * the parameter to cp_set() specifies a full path, that is used instead). dvander@27: * The path is specified into two parts: root and directory; the full path dvander@27: * for the codepage direcory is just the concatenation of the two, with a dvander@27: * directory separator in between. The directory is given in two parts, dvander@27: * because often a program already retrieves its "home" directory and the dvander@27: * codepages are most conveniently stored in a subdirectory of this home dvander@27: * directory. dvander@27: */ dvander@27: SC_FUNC int cp_path(const char *root, const char *directory) dvander@27: { dvander@27: size_t len1,len2; dvander@27: int add_slash1,add_slash2; dvander@27: dvander@27: len1= (root!=NULL) ? strlen(root) : 0; dvander@27: add_slash1= (len1==0 || root[len1-1]!=DIRSEP_CHAR); dvander@27: len2= (directory!=NULL) ? strlen(directory) : 0; dvander@27: add_slash2= (len2>0 && root[len2-1]!=DIRSEP_CHAR); dvander@27: if (len1+add_slash1+len2+add_slash2>=(_MAX_PATH-MAXCODEPAGE)) dvander@27: return FALSE; /* full filename may not fit */ dvander@27: if (root!=NULL) dvander@27: strcpy(cprootpath,root); dvander@27: if (add_slash1) { dvander@27: assert(len1==0 || cprootpath[len1]=='\0'); dvander@27: cprootpath[len1]=DIRSEP_CHAR; dvander@27: cprootpath[len1+1]='\0'; dvander@27: } /* if */ dvander@27: if (directory!=NULL) dvander@27: strcat(cprootpath,directory); dvander@27: if (add_slash2) { dvander@27: assert(cprootpath[len1+add_slash1+len2]=='\0'); dvander@27: cprootpath[len1+add_slash1+len2]=DIRSEP_CHAR; dvander@27: cprootpath[len1+add_slash1+len2+1]='\0'; dvander@27: } /* if */ dvander@27: cp_set(NULL); /* start with a "linear" table (no translation) */ dvander@27: return TRUE; dvander@27: } dvander@27: dvander@27: /* cp_set() loads a codepage from a file. The name parameter may be a dvander@27: * filename (including a full path) or it may be a partial codepage name. dvander@27: * If the name parameter is NULL, the codepage is cleared to be a "linear" dvander@27: * table (no translation). dvander@27: * The following files are attempted to open (where specifies the dvander@27: * value of the parameter): dvander@27: * dvander@27: * / dvander@27: * /.txt dvander@27: * /cp dvander@27: * /cp.txt dvander@27: */ dvander@27: SC_FUNC int cp_set(const char *name) dvander@27: { dvander@27: char filename[_MAX_PATH]; dvander@27: FILE *fp=NULL; dvander@27: unsigned index; dvander@27: dvander@27: /* for name==NULL, set up an identity table */ dvander@27: if (name==NULL || *name=='\0') { dvander@27: if (wordtable!=NULL) { dvander@27: free(wordtable); dvander@27: wordtable=NULL; dvander@27: wordtablesize=0; dvander@27: wordtabletop=0; dvander@27: } /* if */ dvander@27: for (index=0; indexMAXCODEPAGE) dvander@27: return 0; dvander@27: assert(strlen(name)+strlen(cprootpath)<_MAX_PATH); dvander@27: strcpy(filename,cprootpath); dvander@27: strcat(filename,name); dvander@27: fp=fopen(filename,"rt"); dvander@27: } /* if */ dvander@27: if (fp==NULL) { dvander@27: /* try opening the file in the "root path" for codepages, with a ".txt" extension */ dvander@27: if (strlen(name)+4>=MAXCODEPAGE) dvander@27: return 0; dvander@27: assert(strlen(filename)+4<_MAX_PATH); dvander@27: strcat(filename,".txt"); dvander@27: fp=fopen(filename,"rt"); dvander@27: } /* if */ dvander@27: if (fp==NULL) { dvander@27: /* try opening the file in the "root path" for codepages, with "cp" prefixed before the name */ dvander@27: if (strlen(name)+2>MAXCODEPAGE) dvander@27: return 0; dvander@27: assert(2+strlen(name)+strlen(cprootpath)<_MAX_PATH); dvander@27: strcpy(filename,cprootpath); dvander@27: strcat(filename,"cp"); dvander@27: strcat(filename,name); dvander@27: fp=fopen(filename,"rt"); dvander@27: } /* if */ dvander@27: if (fp==NULL) { dvander@27: /* try opening the file in the "root path" for codepages, with "cp" prefixed an ".txt" appended */ dvander@27: if (strlen(name)+2+4>MAXCODEPAGE) dvander@27: return 0; dvander@27: assert(strlen(filename)+4<_MAX_PATH); dvander@27: strcat(filename,".txt"); dvander@27: fp=fopen(filename,"rt"); dvander@27: } /* if */ dvander@27: if (fp==NULL) dvander@27: return FALSE; /* all failed */ dvander@27: dvander@27: /* clear the tables */ dvander@27: for (index=0; index0 && wordtable!=NULL); dvander@27: if (wordtable!=NULL) { dvander@27: free(wordtable); dvander@27: wordtable=NULL; dvander@27: wordtablesize=0; dvander@27: wordtabletop=0; dvander@27: } /* if */ dvander@27: dvander@27: /* read in the table */ dvander@27: while (cp_readline(fp,filename,sizeof filename)) { dvander@27: char *ptr; dvander@27: if ((ptr=strchr(filename,'#'))!=NULL) dvander@27: *ptr='\0'; /* strip of comment */ dvander@27: for (ptr=filename; *ptr>0 && *ptr<' '; ptr++) dvander@27: /* nothing */; /* skip leading whitespace */ dvander@27: if (*ptr!='\0') { dvander@27: /* content on line */ dvander@27: unsigned code=LEADBYTE; dvander@27: int num=sscanf(ptr,"%i %i",&index,&code); dvander@27: /* if sscanf() returns 1 and the index is in range 0..255, then the dvander@27: * code is a DBCS lead byte; if sscanf() returns 2 and index>=256, this dvander@27: * is a double byte pair (lead byte + follower) dvander@27: */ dvander@27: if (num>=1 && index<256) { dvander@27: bytetable[index]=(wchar_t)code; dvander@27: } else if (num==2 && index>=256 && index=wordtablesize) { dvander@27: /* grow the list */ dvander@27: int newsize; dvander@27: struct wordpair *newblock; dvander@27: newsize= (wordtablesize==0) ? 128 : 2*wordtablesize; dvander@27: newblock=(struct wordpair *)malloc(newsize*sizeof(*wordtable)); dvander@27: if (newblock!=NULL) { dvander@27: memcpy(newblock,wordtable,wordtabletop*sizeof(*wordtable)); dvander@27: free(wordtable); dvander@27: wordtable=newblock; dvander@27: wordtablesize=newsize; dvander@27: } /* if */ dvander@27: } /* if */ dvander@27: if (wordtabletop0 && (unsigned)wordtable[pos-1].index>index) { dvander@27: wordtable[pos]=wordtable[pos-1]; dvander@27: pos--; dvander@27: } /* while */ dvander@27: wordtable[pos].index=(unsigned short)index; dvander@27: wordtable[pos].code=(wchar_t)code; dvander@27: } /* if */ dvander@27: } /* if */ dvander@27: } /* if */ dvander@27: } /* while */ dvander@27: dvander@27: fclose(fp); dvander@27: return TRUE; dvander@27: } dvander@27: dvander@27: SC_FUNC cell cp_translate(const unsigned char *string,const unsigned char **endptr) dvander@27: { dvander@27: wchar_t result; dvander@27: dvander@27: result=bytetable[*string++]; dvander@27: /* check whether this is a leader code */ dvander@27: if ((unsigned)result==LEADBYTE && wordtable!=NULL) { dvander@27: /* look up the code via binary search */ dvander@27: int low,high,mid; dvander@27: unsigned short index=(unsigned short)(((*(string-1)) << 8) | *string); dvander@27: string++; dvander@27: assert(wordtabletop>0); dvander@27: low=0; dvander@27: high=wordtabletop-1; dvander@27: while (lowwordtable[mid].index) dvander@27: low=mid+1; dvander@27: else dvander@27: high=mid; dvander@27: } /* while */ dvander@27: assert(low==high); dvander@27: if (wordtable[low].index==index) dvander@27: result=wordtable[low].code; dvander@27: } /* if */ dvander@27: dvander@27: if (endptr!=NULL) dvander@27: *endptr=string; dvander@27: return (cell)result; dvander@27: } dvander@27: dvander@27: #endif /* NO_CODEPAGE */ dvander@27: dvander@27: #if !defined NO_UTF8 dvander@27: SC_FUNC cell get_utf8_char(const unsigned char *string,const unsigned char **endptr) dvander@27: { dvander@27: int follow=0; dvander@27: long lowmark=0; dvander@27: unsigned char ch; dvander@27: cell result=0; dvander@27: dvander@27: if (endptr!=NULL) dvander@27: *endptr=string; dvander@27: dvander@27: for ( ;; ) { dvander@27: ch=*string++; dvander@27: dvander@27: if (follow>0 && (ch & 0xc0)==0x80) { dvander@27: /* leader code is active, combine with earlier code */ dvander@27: result=(result << 6) | (ch & 0x3f); dvander@27: if (--follow==0) { dvander@27: /* encoding a character in more bytes than is strictly needed, dvander@27: * is not really valid UTF-8; we are strict here to increase dvander@27: * the chance of heuristic dectection of non-UTF-8 text dvander@27: * (JAVA writes zero bytes as a 2-byte code UTF-8, which is invalid) dvander@27: */ dvander@27: if (result=0xd800 && result<=0xdfff || result==0xfffe || result==0xffff) dvander@27: return -1; dvander@27: } /* if */ dvander@27: break; dvander@27: } else if (follow==0 && (ch & 0x80)==0x80) { dvander@27: /* UTF-8 leader code */ dvander@27: if ((ch & 0xe0)==0xc0) { dvander@27: /* 110xxxxx 10xxxxxx */ dvander@27: follow=1; dvander@27: lowmark=0x80L; dvander@27: result=ch & 0x1f; dvander@27: } else if ((ch & 0xf0)==0xe0) { dvander@27: /* 1110xxxx 10xxxxxx 10xxxxxx (16 bits, BMP plane) */ dvander@27: follow=2; dvander@27: lowmark=0x800L; dvander@27: result=ch & 0x0f; dvander@27: } else if ((ch & 0xf8)==0xf0) { dvander@27: /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ dvander@27: follow=3; dvander@27: lowmark=0x10000L; dvander@27: result=ch & 0x07; dvander@27: } else if ((ch & 0xfc)==0xf8) { dvander@27: /* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */ dvander@27: follow=4; dvander@27: lowmark=0x200000L; dvander@27: result=ch & 0x03; dvander@27: } else if ((ch & 0xfe)==0xfc) { dvander@27: /* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx (32 bits) */ dvander@27: follow=5; dvander@27: lowmark=0x4000000L; dvander@27: result=ch & 0x01; dvander@27: } else { dvander@27: /* this is invalid UTF-8 */ dvander@27: return -1; dvander@27: } /* if */ dvander@27: } else if (follow==0 && (ch & 0x80)==0x00) { dvander@27: /* 0xxxxxxx (US-ASCII) */ dvander@27: result=ch; dvander@27: break; dvander@27: } else { dvander@27: /* this is invalid UTF-8 */ dvander@27: return -1; dvander@27: } /* if */ dvander@27: dvander@27: } /* for */ dvander@27: dvander@27: if (endptr!=NULL) dvander@27: *endptr=string; dvander@27: return result; dvander@27: } dvander@27: #endif dvander@27: dvander@27: SC_FUNC int scan_utf8(FILE *fp,const char *filename) dvander@27: { dvander@27: #if defined NO_UTF8 dvander@27: return 0; dvander@27: #else dvander@27: void *resetpos=pc_getpossrc(fp); dvander@27: int utf8=TRUE; dvander@27: int firstchar=TRUE,bom_found=FALSE; dvander@27: const unsigned char *ptr; dvander@27: dvander@27: while (utf8 && pc_readsrc(fp,pline,sLINEMAX)!=NULL) { dvander@27: ptr=pline; dvander@27: if (firstchar) { dvander@27: /* check whether the very first character on the very first line dvander@27: * starts with a BYTE order mark dvander@27: */ dvander@27: cell c=get_utf8_char(ptr,&ptr); dvander@27: bom_found= (c==0xfeff); dvander@27: utf8= (c>=0); dvander@27: firstchar=FALSE; dvander@27: } /* if */ dvander@27: while (utf8 && *ptr!='\0') dvander@27: utf8= (get_utf8_char(ptr,&ptr)>=0); dvander@27: } /* while */ dvander@27: pc_resetsrc(fp,resetpos); dvander@27: if (bom_found) { dvander@27: unsigned char bom[3]; dvander@27: if (!utf8) dvander@27: error(77,filename); /* malformed UTF-8 encoding */ dvander@27: pc_readsrc(fp,bom,3); dvander@27: assert(bom[0]==0xef && bom[1]==0xbb && bom[2]==0xbf); dvander@27: } /* if */ dvander@27: return utf8; dvander@27: #endif /* NO_UTF8 */ dvander@27: }