diff -r d2377b6bf852 -r bf68e1639dba compiler-init/sci18n.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/compiler-init/sci18n.c	Thu Jul 13 20:03:46 2006 +0000
@@ -0,0 +1,428 @@
+/*  Codepage translation to Unicode, and UTF-8 support
+ *
+ *  The translation is based on codepage mapping files that are distributed
+ *  by the Unicode consortium, see ftp://ftp.unicode.org/Public/MAPPINGS/.
+ *
+ *  Character sets with a maximum of 256 codes are translated via a lookup
+ *  table (these are Single-Byte Character Sets). Character sets like Shift-JIS
+ *  with single-byte characters and multi-byte characters (introduced by a
+ *  leader byte) are split into two tables: the 256-entry lookup table for
+ *  the single-byte characters and an extended table for the multi-byte
+ *  characters. The extended table is allocated dynamically; the lookup table
+ *  is allocated statically, so loading SBCS tables cannot fail (if the tables
+ *  themselves are valid, of course).
+ *
+ *  Copyright (c) ITB CompuPhase, 2004-2006
+ *
+ *  This software is provided "as-is", without any express or implied warranty.
+ *  In no event will the authors be held liable for any damages arising from
+ *  the use of this software.
+ *
+ *  Permission is granted to anyone to use this software for any purpose,
+ *  including commercial applications, and to alter it and redistribute it
+ *  freely, subject to the following restrictions:
+ *
+ *  1.  The origin of this software must not be misrepresented; you must not
+ *      claim that you wrote the original software. If you use this software in
+ *      a product, an acknowledgment in the product documentation would be
+ *      appreciated but is not required.
+ *  2.  Altered source versions must be plainly marked as such, and must not be
+ *      misrepresented as being the original software.
+ *  3.  This notice may not be removed or altered from any source distribution.
+ *
+ *  Version: $Id: CODEPAGE.C,v 1.0 2004-02-18 12:13:04+01 thiadmer Exp thiadmer $
+ */
+#include <assert.h>
+#include <stdio.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sc.h"
+
+#if !defined TRUE
+  #define FALSE         0
+  #define TRUE          1
+#endif
+#if !defined _MAX_PATH
+  #define _MAX_PATH     250
+#endif
+#if !defined DIRSEP_CHAR
+  #if defined LINUX || defined __FreeBSD__ || defined __OpenBSD__
+    #define DIRSEP_CHAR '/'
+  #elif defined macintosh
+    #define DIRSEP_CHAR ':'
+  #else
+    #define DIRSEP_CHAR '\\'
+  #endif
+#endif
+
+#if !defined ELEMENTS
+  #define ELEMENTS(array)       (sizeof(array) / sizeof(array[0]))
+#endif
+
+#if !defined NO_CODEPAGE
+
+#if !defined MAXCODEPAGE
+  #define MAXCODEPAGE   12      /* typically "cp" + 4 digits + ".txt" */
+#endif
+#define INVALID         0xffffu /* 0xffff and 0xfffe are invalid Unicode characters */
+#define LEADBYTE        0xfffeu
+
+struct wordpair {
+  unsigned short index;
+  wchar_t code;
+};
+static char cprootpath[_MAX_PATH] = { DIRSEP_CHAR, '\0' };
+static wchar_t bytetable[256];
+static struct wordpair *wordtable = NULL;
+static unsigned wordtablesize = 0;
+static unsigned wordtabletop = 0;
+
+
+/* read in a line delimited by '\r' or '\n'; do NOT store the '\r' or '\n' into
+ * the string and ignore empty lines
+ * returns 1 for success and 0 for failure
+ */
+static int cp_readline(FILE *fp,char *string,size_t size)
+{
+  size_t count=0;
+  int c;
+  assert(size>1);
+  while ((c=fgetc(fp))!=EOF && count<size-1) {
+    if (c=='\r' || c=='\n') {
+      if (count>0)  /* '\r' or '\n' ends a string */
+        break;
+      /* if count==0, the line started with a '\r' or '\n', or perhaps line
+       * ends in the file are '\r\n' and we read and stopped on the '\r' of
+       * the preceding line
+       */
+    } else {
+      string[count++]=(char)c;
+    } /* if */
+  } /* while */
+  string[count]='\0';
+  return count>0;
+}
+
+/* cp_path() sets the directory where all codepage files must be found (if
+ * the parameter to cp_set() specifies a full path, that is used instead).
+ * The path is specified into two parts: root and directory; the full path
+ * for the codepage direcory is just the concatenation of the two, with a
+ * directory separator in between. The directory is given in two parts,
+ * because often a program already retrieves its "home" directory and the
+ * codepages are most conveniently stored in a subdirectory of this home
+ * directory.
+ */
+SC_FUNC int cp_path(const char *root, const char *directory)
+{
+  size_t len1,len2;
+  int add_slash1,add_slash2;
+
+  len1= (root!=NULL) ? strlen(root) : 0;
+  add_slash1= (len1==0 || root[len1-1]!=DIRSEP_CHAR);
+  len2= (directory!=NULL) ? strlen(directory) : 0;
+  add_slash2= (len2>0 && root[len2-1]!=DIRSEP_CHAR);
+  if (len1+add_slash1+len2+add_slash2>=(_MAX_PATH-MAXCODEPAGE))
+    return FALSE;       /* full filename may not fit */
+  if (root!=NULL)
+    strcpy(cprootpath,root);
+  if (add_slash1) {
+    assert(len1==0 || cprootpath[len1]=='\0');
+    cprootpath[len1]=DIRSEP_CHAR;
+    cprootpath[len1+1]='\0';
+  } /* if */
+  if (directory!=NULL)
+    strcat(cprootpath,directory);
+  if (add_slash2) {
+    assert(cprootpath[len1+add_slash1+len2]=='\0');
+    cprootpath[len1+add_slash1+len2]=DIRSEP_CHAR;
+    cprootpath[len1+add_slash1+len2+1]='\0';
+  } /* if */
+  cp_set(NULL);         /* start with a "linear" table (no translation) */
+  return TRUE;
+}
+
+/* cp_set() loads a codepage from a file. The name parameter may be a
+ * filename (including a full path) or it may be a partial codepage name.
+ * If the name parameter is NULL, the codepage is cleared to be a "linear"
+ * table (no translation).
+ * The following files are attempted to open (where <name> specifies the
+ * value of the parameter):
+ *    <name>
+ *    <cprootpath>/<name>
+ *    <cprootpath>/<name>.txt
+ *    <cprootpath>/cp<name>
+ *    <cprootpath>/cp<name>.txt
+ */
+SC_FUNC int cp_set(const char *name)
+{
+  char filename[_MAX_PATH];
+  FILE *fp=NULL;
+  unsigned index;
+
+  /* for name==NULL, set up an identity table */
+  if (name==NULL || *name=='\0') {
+    if (wordtable!=NULL) {
+      free(wordtable);
+      wordtable=NULL;
+      wordtablesize=0;
+      wordtabletop=0;
+    } /* if */
+    for (index=0; index<ELEMENTS(bytetable); index++)
+      bytetable[index]=(wchar_t)index;
+    return TRUE;
+  } /* if */
+
+  /* try to open the file as-is */
+  if (strchr(name,DIRSEP_CHAR)!=NULL)
+    fp=fopen(name,"rt");
+  if (fp==NULL) {
+    /* try opening the file in the "root path" for codepages */
+    if (strlen(name)>MAXCODEPAGE)
+      return 0;
+    assert(strlen(name)+strlen(cprootpath)<_MAX_PATH);
+    strcpy(filename,cprootpath);
+    strcat(filename,name);
+    fp=fopen(filename,"rt");
+  } /* if */
+  if (fp==NULL) {
+    /* try opening the file in the "root path" for codepages, with a ".txt" extension */
+    if (strlen(name)+4>=MAXCODEPAGE)
+      return 0;
+    assert(strlen(filename)+4<_MAX_PATH);
+    strcat(filename,".txt");
+    fp=fopen(filename,"rt");
+  } /* if */
+  if (fp==NULL) {
+    /* try opening the file in the "root path" for codepages, with "cp" prefixed before the name */
+    if (strlen(name)+2>MAXCODEPAGE)
+      return 0;
+    assert(2+strlen(name)+strlen(cprootpath)<_MAX_PATH);
+    strcpy(filename,cprootpath);
+    strcat(filename,"cp");
+    strcat(filename,name);
+    fp=fopen(filename,"rt");
+  } /* if */
+  if (fp==NULL) {
+    /* try opening the file in the "root path" for codepages, with "cp" prefixed an ".txt" appended */
+    if (strlen(name)+2+4>MAXCODEPAGE)
+      return 0;
+    assert(strlen(filename)+4<_MAX_PATH);
+    strcat(filename,".txt");
+    fp=fopen(filename,"rt");
+  } /* if */
+  if (fp==NULL)
+    return FALSE;       /* all failed */
+
+  /* clear the tables */
+  for (index=0; index<ELEMENTS(bytetable); index++)
+    bytetable[index]=INVALID;   /* special code meaning "not found" */
+  assert(wordtablesize==0 && wordtabletop==0 && wordtable==NULL
+         || wordtablesize>0 && wordtable!=NULL);
+  if (wordtable!=NULL) {
+    free(wordtable);
+    wordtable=NULL;
+    wordtablesize=0;
+    wordtabletop=0;
+  } /* if */
+
+  /* read in the table */
+  while (cp_readline(fp,filename,sizeof filename)) {
+    char *ptr;
+    if ((ptr=strchr(filename,'#'))!=NULL)
+      *ptr='\0';                /* strip of comment */
+    for (ptr=filename; *ptr>0 && *ptr<' '; ptr++)
+      /* nothing */;            /* skip leading whitespace */
+    if (*ptr!='\0') {
+      /* content on line */
+      unsigned code=LEADBYTE;
+      int num=sscanf(ptr,"%i %i",&index,&code);
+      /* if sscanf() returns 1 and the index is in range 0..255, then the
+       * code is a DBCS lead byte; if sscanf() returns 2 and index>=256, this
+       * is a double byte pair (lead byte + follower)
+       */
+      if (num>=1 && index<256) {
+        bytetable[index]=(wchar_t)code;
+      } else if (num==2 && index>=256 && index<LEADBYTE) {
+        /* store the DBCS character in wordtable */
+        if (wordtabletop>=wordtablesize) {
+          /* grow the list */
+          int newsize;
+          struct wordpair *newblock;
+          newsize= (wordtablesize==0) ? 128 : 2*wordtablesize;
+          newblock=(struct wordpair *)malloc(newsize*sizeof(*wordtable));
+          if (newblock!=NULL) {
+            memcpy(newblock,wordtable,wordtabletop*sizeof(*wordtable));
+            free(wordtable);
+            wordtable=newblock;
+            wordtablesize=newsize;
+          } /* if */
+        } /* if */
+        if (wordtabletop<wordtablesize) {
+          /* insert at sorted position */
+          int pos=wordtabletop;
+          assert(wordtable!=NULL);
+          while (pos>0 && (unsigned)wordtable[pos-1].index>index) {
+            wordtable[pos]=wordtable[pos-1];
+            pos--;
+          } /* while */
+          wordtable[pos].index=(unsigned short)index;
+          wordtable[pos].code=(wchar_t)code;
+        } /* if */
+      } /* if */
+    } /* if */
+  } /* while */
+
+  fclose(fp);
+  return TRUE;
+}
+
+SC_FUNC cell cp_translate(const unsigned char *string,const unsigned char **endptr)
+{
+  wchar_t result;
+
+  result=bytetable[*string++];
+  /* check whether this is a leader code */
+  if ((unsigned)result==LEADBYTE && wordtable!=NULL) {
+    /* look up the code via binary search */
+    int low,high,mid;
+    unsigned short index=(unsigned short)(((*(string-1)) << 8) | *string);
+    string++;
+    assert(wordtabletop>0);
+    low=0;
+    high=wordtabletop-1;
+    while (low<high) {
+      mid=(low+high)/2;
+      assert(low<=mid && mid<high);
+      if (index>wordtable[mid].index)
+        low=mid+1;
+      else
+        high=mid;
+    } /* while */
+    assert(low==high);
+    if (wordtable[low].index==index)
+      result=wordtable[low].code;
+  } /* if */
+
+  if (endptr!=NULL)
+    *endptr=string;
+  return (cell)result;
+}
+
+#endif  /* NO_CODEPAGE */
+
+#if !defined NO_UTF8
+SC_FUNC cell get_utf8_char(const unsigned char *string,const unsigned char **endptr)
+{
+  int follow=0;
+  long lowmark=0;
+  unsigned char ch;
+  cell result=0;
+
+  if (endptr!=NULL)
+    *endptr=string;
+
+  for ( ;; ) {
+    ch=*string++;
+
+    if (follow>0 && (ch & 0xc0)==0x80) {
+      /* leader code is active, combine with earlier code */
+      result=(result << 6) | (ch & 0x3f);
+      if (--follow==0) {
+        /* encoding a character in more bytes than is strictly needed,
+         * is not really valid UTF-8; we are strict here to increase
+         * the chance of heuristic dectection of non-UTF-8 text
+         * (JAVA writes zero bytes as a 2-byte code UTF-8, which is invalid)
+         */
+        if (result<lowmark)
+          return -1;
+        /* the code positions 0xd800--0xdfff and 0xfffe & 0xffff do not
+         * exist in UCS-4 (and hence, they do not exist in Unicode)
+         */
+        if (result>=0xd800 && result<=0xdfff || result==0xfffe || result==0xffff)
+          return -1;
+      } /* if */
+      break;
+    } else if (follow==0 && (ch & 0x80)==0x80) {
+      /* UTF-8 leader code */
+      if ((ch & 0xe0)==0xc0) {
+        /* 110xxxxx 10xxxxxx */
+        follow=1;
+        lowmark=0x80L;
+        result=ch & 0x1f;
+      } else if ((ch & 0xf0)==0xe0) {
+        /* 1110xxxx 10xxxxxx 10xxxxxx (16 bits, BMP plane) */
+        follow=2;
+        lowmark=0x800L;
+        result=ch & 0x0f;
+      } else if ((ch & 0xf8)==0xf0) {
+        /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
+        follow=3;
+        lowmark=0x10000L;
+        result=ch & 0x07;
+      } else if ((ch & 0xfc)==0xf8) {
+        /* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */
+        follow=4;
+        lowmark=0x200000L;
+        result=ch & 0x03;
+      } else if ((ch & 0xfe)==0xfc) {
+        /* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx (32 bits) */
+        follow=5;
+        lowmark=0x4000000L;
+        result=ch & 0x01;
+      } else {
+        /* this is invalid UTF-8 */
+        return -1;
+      } /* if */
+    } else if (follow==0 && (ch & 0x80)==0x00) {
+      /* 0xxxxxxx (US-ASCII) */
+      result=ch;
+      break;
+    } else {
+      /* this is invalid UTF-8 */
+      return -1;
+    } /* if */
+
+  } /* for */
+
+  if (endptr!=NULL)
+    *endptr=string;
+  return result;
+}
+#endif
+
+SC_FUNC int scan_utf8(FILE *fp,const char *filename)
+{
+  #if defined NO_UTF8
+    return 0;
+  #else
+    void *resetpos=pc_getpossrc(fp);
+    int utf8=TRUE;
+    int firstchar=TRUE,bom_found=FALSE;
+    const unsigned char *ptr;
+
+    while (utf8 && pc_readsrc(fp,pline,sLINEMAX)!=NULL) {
+      ptr=pline;
+      if (firstchar) {
+        /* check whether the very first character on the very first line
+         * starts with a BYTE order mark
+         */
+        cell c=get_utf8_char(ptr,&ptr);
+        bom_found= (c==0xfeff);
+        utf8= (c>=0);
+        firstchar=FALSE;
+      } /* if */
+      while (utf8 && *ptr!='\0')
+        utf8= (get_utf8_char(ptr,&ptr)>=0);
+    } /* while */
+    pc_resetsrc(fp,resetpos);
+    if (bom_found) {
+      unsigned char bom[3];
+      if (!utf8)
+        error(77,filename);     /* malformed UTF-8 encoding */
+      pc_readsrc(fp,bom,3);
+      assert(bom[0]==0xef && bom[1]==0xbb && bom[2]==0xbf);
+    } /* if */
+    return utf8;
+  #endif  /* NO_UTF8 */
+}