/*  Codepage translation to Unicode, and UTF-8 support

 *

 *  The translation is based on codepage mapping files that are distributed

 *  by the Unicode consortium, see ftp://ftp.unicode.org/Public/MAPPINGS/.

 *

 *  Character sets with a maximum of 256 codes are translated via a lookup

 *  table (these are Single-Byte Character Sets). Character sets like Shift-JIS

 *  with single-byte characters and multi-byte characters (introduced by a

 *  leader byte) are split into two tables: the 256-entry lookup table for

 *  the single-byte characters and an extended table for the multi-byte

 *  characters. The extended table is allocated dynamically; the lookup table

 *  is allocated statically, so loading SBCS tables cannot fail (if the tables

 *  themselves are valid, of course).

 *

 *  Copyright (c) ITB CompuPhase, 2004-2006

 *

 *  This software is provided "as-is", without any express or implied warranty.

 *  In no event will the authors be held liable for any damages arising from

 *  the use of this software.

 *

 *  Permission is granted to anyone to use this software for any purpose,

 *  including commercial applications, and to alter it and redistribute it

 *  freely, subject to the following restrictions:

 *

 *  1.  The origin of this software must not be misrepresented; you must not

 *      claim that you wrote the original software. If you use this software in

 *      a product, an acknowledgment in the product documentation would be

 *      appreciated but is not required.

 *  2.  Altered source versions must be plainly marked as such, and must not be

 *      misrepresented as being the original software.

 *  3.  This notice may not be removed or altered from any source distribution.

 *

 *  Version: $Id: CODEPAGE.C,v 1.0 2004-02-18 12:13:04+01 thiadmer Exp thiadmer $

 */

#include <assert.h>

#include <stdio.h>

#include <stddef.h>

#include <stdlib.h>

#include <string.h>

#include "sc.h"

#if !defined TRUE

  #define FALSE         0

  #define TRUE          1

#endif

#if !defined _MAX_PATH

  #define _MAX_PATH     250

#endif

#if !defined DIRSEP_CHAR

  #if defined LINUX || defined __FreeBSD__ || defined __OpenBSD__

    #define DIRSEP_CHAR '/'

  #elif defined macintosh

    #define DIRSEP_CHAR ':'

  #else

    #define DIRSEP_CHAR '\\'

  #endif

#endif

#if !defined ELEMENTS

  #define ELEMENTS(array)       (sizeof(array) / sizeof(array[0]))

#endif

#if !defined NO_CODEPAGE

#if !defined MAXCODEPAGE

  #define MAXCODEPAGE   12      /* typically "cp" + 4 digits + ".txt" */

#endif

#define INVALID         0xffffu /* 0xffff and 0xfffe are invalid Unicode characters */

#define LEADBYTE        0xfffeu

struct wordpair {

  unsigned short index;

  wchar_t code;

};

static char cprootpath[_MAX_PATH] = { DIRSEP_CHAR, '\0' };

static wchar_t bytetable[256];

static struct wordpair *wordtable = NULL;

static unsigned wordtablesize = 0;

static unsigned wordtabletop = 0;

/* read in a line delimited by '\r' or '\n'; do NOT store the '\r' or '\n' into

 * the string and ignore empty lines

 * returns 1 for success and 0 for failure

 */

static int cp_readline(FILE *fp,char *string,size_t size)

{

  size_t count=0;

  int c;

  assert(size>1);

  while ((c=fgetc(fp))!=EOF && count<size-1) {

    if (c=='\r' || c=='\n') {

      if (count>0)  /* '\r' or '\n' ends a string */

        break;

      /* if count==0, the line started with a '\r' or '\n', or perhaps line

       * ends in the file are '\r\n' and we read and stopped on the '\r' of

       * the preceding line

       */

    } else {

      string[count++]=(char)c;

    } /* if */

  } /* while */

  string[count]='\0';

  return count>0;

}

/* cp_path() sets the directory where all codepage files must be found (if

 * the parameter to cp_set() specifies a full path, that is used instead).

 * The path is specified into two parts: root and directory; the full path

 * for the codepage direcory is just the concatenation of the two, with a

 * directory separator in between. The directory is given in two parts,

 * because often a program already retrieves its "home" directory and the

 * codepages are most conveniently stored in a subdirectory of this home

 * directory.

 */

SC_FUNC int cp_path(const char *root, const char *directory)

{

  size_t len1,len2;

  int add_slash1,add_slash2;

  len1= (root!=NULL) ? strlen(root) : 0;

  add_slash1= (len1==0 || root[len1-1]!=DIRSEP_CHAR);

  len2= (directory!=NULL) ? strlen(directory) : 0;

  add_slash2= (len2>0 && root[len2-1]!=DIRSEP_CHAR);

  if (len1+add_slash1+len2+add_slash2>=(_MAX_PATH-MAXCODEPAGE))

    return FALSE;       /* full filename may not fit */

  if (root!=NULL)

    strcpy(cprootpath,root);

  if (add_slash1) {

    assert(len1==0 || cprootpath[len1]=='\0');

    cprootpath[len1]=DIRSEP_CHAR;

    cprootpath[len1+1]='\0';

  } /* if */

  if (directory!=NULL)

    strcat(cprootpath,directory);

  if (add_slash2) {

    assert(cprootpath[len1+add_slash1+len2]=='\0');

    cprootpath[len1+add_slash1+len2]=DIRSEP_CHAR;

    cprootpath[len1+add_slash1+len2+1]='\0';

  } /* if */

  cp_set(NULL);         /* start with a "linear" table (no translation) */

  return TRUE;

}

/* cp_set() loads a codepage from a file. The name parameter may be a

 * filename (including a full path) or it may be a partial codepage name.

 * If the name parameter is NULL, the codepage is cleared to be a "linear"

 * table (no translation).

 * The following files are attempted to open (where <name> specifies the

 * value of the parameter):

 *    <name>

 *    <cprootpath>/<name>

 *    <cprootpath>/<name>.txt

 *    <cprootpath>/cp<name>

 *    <cprootpath>/cp<name>.txt

 */

SC_FUNC int cp_set(const char *name)

{

  char filename[_MAX_PATH];

  FILE *fp=NULL;

  unsigned index;

  /* for name==NULL, set up an identity table */

  if (name==NULL || *name=='\0') {

    if (wordtable!=NULL) {

      free(wordtable);

      wordtable=NULL;

      wordtablesize=0;

      wordtabletop=0;

    } /* if */

    for (index=0; index<ELEMENTS(bytetable); index++)

      bytetable[index]=(wchar_t)index;

    return TRUE;

  } /* if */

  /* try to open the file as-is */

  if (strchr(name,DIRSEP_CHAR)!=NULL)

    fp=fopen(name,"rt");

  if (fp==NULL) {

    /* try opening the file in the "root path" for codepages */

    if (strlen(name)>MAXCODEPAGE)

      return 0;

    assert(strlen(name)+strlen(cprootpath)<_MAX_PATH);

    strcpy(filename,cprootpath);

    strcat(filename,name);

    fp=fopen(filename,"rt");

  } /* if */

  if (fp==NULL) {

    /* try opening the file in the "root path" for codepages, with a ".txt" extension */

    if (strlen(name)+4>=MAXCODEPAGE)

      return 0;

    assert(strlen(filename)+4<_MAX_PATH);

    strcat(filename,".txt");

    fp=fopen(filename,"rt");

  } /* if */

  if (fp==NULL) {

    /* try opening the file in the "root path" for codepages, with "cp" prefixed before the name */

    if (strlen(name)+2>MAXCODEPAGE)

      return 0;

    assert(2+strlen(name)+strlen(cprootpath)<_MAX_PATH);

    strcpy(filename,cprootpath);

    strcat(filename,"cp");

    strcat(filename,name);

    fp=fopen(filename,"rt");

  } /* if */

  if (fp==NULL) {

    /* try opening the file in the "root path" for codepages, with "cp" prefixed an ".txt" appended */

    if (strlen(name)+2+4>MAXCODEPAGE)

      return 0;

    assert(strlen(filename)+4<_MAX_PATH);

    strcat(filename,".txt");

    fp=fopen(filename,"rt");

  } /* if */

  if (fp==NULL)

    return FALSE;       /* all failed */

  /* clear the tables */

  for (index=0; index<ELEMENTS(bytetable); index++)

    bytetable[index]=INVALID;   /* special code meaning "not found" */

  assert(wordtablesize==0 && wordtabletop==0 && wordtable==NULL

         || wordtablesize>0 && wordtable!=NULL);

  if (wordtable!=NULL) {

    free(wordtable);

    wordtable=NULL;

    wordtablesize=0;

    wordtabletop=0;

  } /* if */

  /* read in the table */

  while (cp_readline(fp,filename,sizeof filename)) {

    char *ptr;

    if ((ptr=strchr(filename,'#'))!=NULL)

      *ptr='\0';                /* strip of comment */

    for (ptr=filename; *ptr>0 && *ptr<' '; ptr++)

      /* nothing */;            /* skip leading whitespace */

    if (*ptr!='\0') {

      /* content on line */

      unsigned code=LEADBYTE;

      int num=sscanf(ptr,"%i %i",&index,&code);

      /* if sscanf() returns 1 and the index is in range 0..255, then the

       * code is a DBCS lead byte; if sscanf() returns 2 and index>=256, this

       * is a double byte pair (lead byte + follower)

       */

      if (num>=1 && index<256) {

        bytetable[index]=(wchar_t)code;

      } else if (num==2 && index>=256 && index<LEADBYTE) {

        /* store the DBCS character in wordtable */

        if (wordtabletop>=wordtablesize) {

          /* grow the list */

          int newsize;

          struct wordpair *newblock;

          newsize= (wordtablesize==0) ? 128 : 2*wordtablesize;

          newblock=(struct wordpair *)malloc(newsize*sizeof(*wordtable));

          if (newblock!=NULL) {

            memcpy(newblock,wordtable,wordtabletop*sizeof(*wordtable));

            free(wordtable);

            wordtable=newblock;

            wordtablesize=newsize;

          } /* if */

        } /* if */

        if (wordtabletop<wordtablesize) {

          /* insert at sorted position */

          int pos=wordtabletop;

          assert(wordtable!=NULL);

          while (pos>0 && (unsigned)wordtable[pos-1].index>index) {

            wordtable[pos]=wordtable[pos-1];

            pos--;

          } /* while */

          wordtable[pos].index=(unsigned short)index;

          wordtable[pos].code=(wchar_t)code;

        } /* if */

      } /* if */

    } /* if */

  } /* while */

  fclose(fp);

  return TRUE;

}

SC_FUNC cell cp_translate(const unsigned char *string,const unsigned char **endptr)

{

  wchar_t result;

  result=bytetable[*string++];

  /* check whether this is a leader code */

  if ((unsigned)result==LEADBYTE && wordtable!=NULL) {

    /* look up the code via binary search */

    int low,high,mid;

    unsigned short index=(unsigned short)(((*(string-1)) << 8) | *string);

    string++;

    assert(wordtabletop>0);

    low=0;

    high=wordtabletop-1;

    while (low<high) {

      mid=(low+high)/2;

      assert(low<=mid && mid<high);

      if (index>wordtable[mid].index)

        low=mid+1;

      else

        high=mid;

    } /* while */

    assert(low==high);

    if (wordtable[low].index==index)

      result=wordtable[low].code;

  } /* if */

  if (endptr!=NULL)

    *endptr=string;

  return (cell)result;

}

#endif  /* NO_CODEPAGE */

#if !defined NO_UTF8

SC_FUNC cell get_utf8_char(const unsigned char *string,const unsigned char **endptr)

{

  int follow=0;

  long lowmark=0;

  unsigned char ch;

  cell result=0;

  if (endptr!=NULL)

    *endptr=string;

  for ( ;; ) {

    ch=*string++;

    if (follow>0 && (ch & 0xc0)==0x80) {

      /* leader code is active, combine with earlier code */

      result=(result << 6) | (ch & 0x3f);

      if (--follow==0) {

        /* encoding a character in more bytes than is strictly needed,

         * is not really valid UTF-8; we are strict here to increase

         * the chance of heuristic dectection of non-UTF-8 text

         * (JAVA writes zero bytes as a 2-byte code UTF-8, which is invalid)

         */

        if (result<lowmark)

          return -1;

        /* the code positions 0xd800--0xdfff and 0xfffe & 0xffff do not

         * exist in UCS-4 (and hence, they do not exist in Unicode)

         */

        if (result>=0xd800 && result<=0xdfff || result==0xfffe || result==0xffff)

          return -1;

      } /* if */

      break;

    } else if (follow==0 && (ch & 0x80)==0x80) {

      /* UTF-8 leader code */

      if ((ch & 0xe0)==0xc0) {

        /* 110xxxxx 10xxxxxx */

        follow=1;

        lowmark=0x80L;

        result=ch & 0x1f;

      } else if ((ch & 0xf0)==0xe0) {

        /* 1110xxxx 10xxxxxx 10xxxxxx (16 bits, BMP plane) */

        follow=2;

        lowmark=0x800L;

        result=ch & 0x0f;

      } else if ((ch & 0xf8)==0xf0) {

        /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */

        follow=3;

        lowmark=0x10000L;

        result=ch & 0x07;

      } else if ((ch & 0xfc)==0xf8) {

        /* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */

        follow=4;

        lowmark=0x200000L;

        result=ch & 0x03;

      } else if ((ch & 0xfe)==0xfc) {

        /* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx (32 bits) */

        follow=5;

        lowmark=0x4000000L;

        result=ch & 0x01;

      } else {

        /* this is invalid UTF-8 */

        return -1;

      } /* if */

    } else if (follow==0 && (ch & 0x80)==0x00) {

      /* 0xxxxxxx (US-ASCII) */

      result=ch;

      break;

    } else {

      /* this is invalid UTF-8 */

      return -1;

    } /* if */

  } /* for */

  if (endptr!=NULL)

    *endptr=string;

  return result;

}

#endif

SC_FUNC int scan_utf8(FILE *fp,const char *filename)

{

  #if defined NO_UTF8

    return 0;

  #else

    void *resetpos=pc_getpossrc(fp);

    int utf8=TRUE;

    int firstchar=TRUE,bom_found=FALSE;

    const unsigned char *ptr;

    while (utf8 && pc_readsrc(fp,pline,sLINEMAX)!=NULL) {

      ptr=pline;

      if (firstchar) {

        /* check whether the very first character on the very first line

         * starts with a BYTE order mark

         */

        cell c=get_utf8_char(ptr,&ptr);

        bom_found= (c==0xfeff);

        utf8= (c>=0);

        firstchar=FALSE;

      } /* if */

      while (utf8 && *ptr!='\0')

        utf8= (get_utf8_char(ptr,&ptr)>=0);

    } /* while */

    pc_resetsrc(fp,resetpos);

    if (bom_found) {

      unsigned char bom[3];

      if (!utf8)

        error(77,filename);     /* malformed UTF-8 encoding */

      pc_readsrc(fp,bom,3);

      assert(bom[0]==0xef && bom[1]==0xbb && bom[2]==0xbf);

    } /* if */

    return utf8;

  #endif  /* NO_UTF8 */

}