1.1 --- a/extensions/sqlite/sqlite-source/utf.c Sun Oct 28 05:06:12 2007 +0000
1.2 +++ b/extensions/sqlite/sqlite-source/utf.c Sun Oct 28 05:48:06 2007 +0000
1.3 @@ -49,7 +49,7 @@
1.4 ** This lookup table is used to help decode the first byte of
1.5 ** a multi-byte UTF8 character.
1.6 */
1.7 -const unsigned char sqlite3UtfTrans1[] = {
1.8 +static const unsigned char sqlite3UtfTrans1[] = {
1.9 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
1.10 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
1.11 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
1.12 @@ -60,6 +60,7 @@
1.13 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
1.14 };
1.15
1.16 +
1.17 #define WRITE_UTF8(zOut, c) { \
1.18 if( c<0x00080 ){ \
1.19 *zOut++ = (c&0xFF); \
1.20 @@ -127,6 +128,54 @@
1.21 }
1.22
1.23 /*
1.24 +** Translate a single UTF-8 character. Return the unicode value.
1.25 +**
1.26 +** During translation, assume that the byte that zTerm points
1.27 +** is a 0x00.
1.28 +**
1.29 +** Write a pointer to the next unread byte back into *pzNext.
1.30 +**
1.31 +** Notes On Invalid UTF-8:
1.32 +**
1.33 +** * This routine never allows a 7-bit character (0x00 through 0x7f) to
1.34 +** be encoded as a multi-byte character. Any multi-byte character that
1.35 +** attempts to encode a value between 0x00 and 0x7f is rendered as 0xfffd.
1.36 +**
1.37 +** * This routine never allows a UTF16 surrogate value to be encoded.
1.38 +** If a multi-byte character attempts to encode a value between
1.39 +** 0xd800 and 0xe000 then it is rendered as 0xfffd.
1.40 +**
1.41 +** * Bytes in the range of 0x80 through 0xbf which occur as the first
1.42 +** byte of a character are interpreted as single-byte characters
1.43 +** and rendered as themselves even though they are technically
1.44 +** invalid characters.
1.45 +**
1.46 +** * This routine accepts an infinite number of different UTF8 encodings
1.47 +** for unicode values 0x80 and greater. It do not change over-length
1.48 +** encodings to 0xfffd as some systems recommend.
1.49 +*/
1.50 +int sqlite3Utf8Read(
1.51 + const unsigned char *z, /* First byte of UTF-8 character */
1.52 + const unsigned char *zTerm, /* Pretend this byte is 0x00 */
1.53 + const unsigned char **pzNext /* Write first byte past UTF-8 char here */
1.54 +){
1.55 + int c = *(z++);
1.56 + if( c>=0xc0 ){
1.57 + c = sqlite3UtfTrans1[c-0xc0];
1.58 + while( z!=zTerm && (*z & 0xc0)==0x80 ){
1.59 + c = (c<<6) + (0x3f & *(z++));
1.60 + }
1.61 + if( c<0x80
1.62 + || (c&0xFFFFF800)==0xD800
1.63 + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; }
1.64 + }
1.65 + *pzNext = z;
1.66 + return c;
1.67 +}
1.68 +
1.69 +
1.70 +
1.71 +/*
1.72 ** If the TRANSLATE_TRACE macro is defined, the value of each Mem is
1.73 ** printed on stderr on the way into and out of sqlite3VdbeMemTranslate().
1.74 */
1.75 @@ -147,6 +196,7 @@
1.76 unsigned char *z; /* Output iterator */
1.77 unsigned int c;
1.78
1.79 + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) );
1.80 assert( pMem->flags&MEM_Str );
1.81 assert( pMem->enc!=desiredEnc );
1.82 assert( pMem->enc!=0 );
1.83 @@ -205,95 +255,35 @@
1.84 ** byte past the end.
1.85 **
1.86 ** Variable zOut is set to point at the output buffer. This may be space
1.87 - ** obtained from malloc(), or Mem.zShort, if it large enough and not in
1.88 - ** use, or the zShort array on the stack (see above).
1.89 + ** obtained from sqlite3_malloc(), or Mem.zShort, if it large enough and
1.90 + ** not in use, or the zShort array on the stack (see above).
1.91 */
1.92 zIn = (u8*)pMem->z;
1.93 zTerm = &zIn[pMem->n];
1.94 if( len>NBFS ){
1.95 - zOut = sqliteMallocRaw(len);
1.96 - if( !zOut ) return SQLITE_NOMEM;
1.97 + zOut = sqlite3DbMallocRaw(pMem->db, len);
1.98 + if( !zOut ){
1.99 + return SQLITE_NOMEM;
1.100 + }
1.101 }else{
1.102 zOut = zShort;
1.103 }
1.104 z = zOut;
1.105
1.106 if( pMem->enc==SQLITE_UTF8 ){
1.107 - unsigned int iExtra = 0xD800;
1.108 -
1.109 - if( 0==(pMem->flags&MEM_Term) && zTerm>zIn && (zTerm[-1]&0x80) ){
1.110 - /* This UTF8 string is not nul-terminated, and the last byte is
1.111 - ** not a character in the ascii range (codpoints 0..127). This
1.112 - ** means the SQLITE_READ_UTF8() macro might read past the end
1.113 - ** of the allocated buffer.
1.114 - **
1.115 - ** There are four possibilities:
1.116 - **
1.117 - ** 1. The last byte is the first byte of a non-ASCII character,
1.118 - **
1.119 - ** 2. The final N bytes of the input string are continuation bytes
1.120 - ** and immediately preceding them is the first byte of a
1.121 - ** non-ASCII character.
1.122 - **
1.123 - ** 3. The final N bytes of the input string are continuation bytes
1.124 - ** and immediately preceding them is a byte that encodes a
1.125 - ** character in the ASCII range.
1.126 - **
1.127 - ** 4. The entire string consists of continuation characters.
1.128 - **
1.129 - ** Cases (3) and (4) require no special handling. The SQLITE_READ_UTF8()
1.130 - ** macro will not overread the buffer in these cases.
1.131 - */
1.132 - unsigned char *zExtra = &zTerm[-1];
1.133 - while( zExtra>zIn && (zExtra[0]&0xC0)==0x80 ){
1.134 - zExtra--;
1.135 - }
1.136 -
1.137 - if( (zExtra[0]&0xC0)==0xC0 ){
1.138 - /* Make a copy of the last character encoding in the input string.
1.139 - ** Then make sure it is nul-terminated and use SQLITE_READ_UTF8()
1.140 - ** to decode the codepoint. Store the codepoint in variable iExtra,
1.141 - ** it will be appended to the output string later.
1.142 - */
1.143 - unsigned char *zFree = 0;
1.144 - unsigned char zBuf[16];
1.145 - int nExtra = (pMem->n+zIn-zExtra);
1.146 - zTerm = zExtra;
1.147 - if( nExtra>15 ){
1.148 - zExtra = sqliteMallocRaw(nExtra+1);
1.149 - if( !zExtra ){
1.150 - return SQLITE_NOMEM;
1.151 - }
1.152 - zFree = zExtra;
1.153 - }else{
1.154 - zExtra = zBuf;
1.155 - }
1.156 - memcpy(zExtra, zTerm, nExtra);
1.157 - zExtra[nExtra] = '\0';
1.158 - SQLITE_READ_UTF8(zExtra, iExtra);
1.159 - sqliteFree(zFree);
1.160 - }
1.161 - }
1.162 -
1.163 if( desiredEnc==SQLITE_UTF16LE ){
1.164 /* UTF-8 -> UTF-16 Little-endian */
1.165 while( zIn<zTerm ){
1.166 - SQLITE_READ_UTF8(zIn, c);
1.167 + c = sqlite3Utf8Read(zIn, zTerm, (const u8**)&zIn);
1.168 WRITE_UTF16LE(z, c);
1.169 }
1.170 - if( iExtra!=0xD800 ){
1.171 - WRITE_UTF16LE(z, iExtra);
1.172 - }
1.173 }else{
1.174 assert( desiredEnc==SQLITE_UTF16BE );
1.175 /* UTF-8 -> UTF-16 Big-endian */
1.176 while( zIn<zTerm ){
1.177 - SQLITE_READ_UTF8(zIn, c);
1.178 + c = sqlite3Utf8Read(zIn, zTerm, (const u8**)&zIn);
1.179 WRITE_UTF16BE(z, c);
1.180 }
1.181 - if( iExtra!=0xD800 ){
1.182 - WRITE_UTF16BE(z, iExtra);
1.183 - }
1.184 }
1.185 pMem->n = z - zOut;
1.186 *z++ = 0;
1.187 @@ -377,7 +367,8 @@
1.188 char *z = pMem->z;
1.189 pMem->z = 0;
1.190 pMem->xDel = 0;
1.191 - rc = sqlite3VdbeMemSetStr(pMem, &z[2], pMem->n-2, bom, SQLITE_TRANSIENT);
1.192 + rc = sqlite3VdbeMemSetStr(pMem, &z[2], pMem->n-2, bom,
1.193 + SQLITE_TRANSIENT);
1.194 xDel(z);
1.195 }else{
1.196 rc = sqlite3VdbeMemSetStr(pMem, &pMem->z[2], pMem->n-2, bom,
1.197 @@ -412,22 +403,57 @@
1.198 return r;
1.199 }
1.200
1.201 +/* This test function is not currently used by the automated test-suite.
1.202 +** Hence it is only available in debug builds.
1.203 +*/
1.204 +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG)
1.205 +/*
1.206 +** Translate UTF-8 to UTF-8.
1.207 +**
1.208 +** This has the effect of making sure that the string is well-formed
1.209 +** UTF-8. Miscoded characters are removed.
1.210 +**
1.211 +** The translation is done in-place (since it is impossible for the
1.212 +** correct UTF-8 encoding to be longer than a malformed encoding).
1.213 +*/
1.214 +int sqlite3Utf8To8(unsigned char *zIn){
1.215 + unsigned char *zOut = zIn;
1.216 + unsigned char *zStart = zIn;
1.217 + unsigned char *zTerm;
1.218 + u32 c;
1.219 +
1.220 + while( zIn[0] ){
1.221 + c = sqlite3Utf8Read(zIn, zTerm, (const u8**)&zIn);
1.222 + if( c!=0xfffd ){
1.223 + WRITE_UTF8(zOut, c);
1.224 + }
1.225 + }
1.226 + *zOut = 0;
1.227 + return zOut - zStart;
1.228 +}
1.229 +#endif
1.230 +
1.231 #ifndef SQLITE_OMIT_UTF16
1.232 /*
1.233 ** Convert a UTF-16 string in the native encoding into a UTF-8 string.
1.234 -** Memory to hold the UTF-8 string is obtained from malloc and must be
1.235 -** freed by the calling function.
1.236 +** Memory to hold the UTF-8 string is obtained from sqlite3_malloc and must
1.237 +** be freed by the calling function.
1.238 **
1.239 ** NULL is returned if there is an allocation error.
1.240 */
1.241 -char *sqlite3Utf16to8(const void *z, int nByte){
1.242 +char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte){
1.243 Mem m;
1.244 memset(&m, 0, sizeof(m));
1.245 + m.db = db;
1.246 sqlite3VdbeMemSetStr(&m, z, nByte, SQLITE_UTF16NATIVE, SQLITE_STATIC);
1.247 sqlite3VdbeChangeEncoding(&m, SQLITE_UTF8);
1.248 - assert( (m.flags & MEM_Term)!=0 || sqlite3MallocFailed() );
1.249 - assert( (m.flags & MEM_Str)!=0 || sqlite3MallocFailed() );
1.250 - return (m.flags & MEM_Dyn)!=0 ? m.z : sqliteStrDup(m.z);
1.251 + if( db->mallocFailed ){
1.252 + sqlite3VdbeMemRelease(&m);
1.253 + m.z = 0;
1.254 + }
1.255 + assert( (m.flags & MEM_Term)!=0 || db->mallocFailed );
1.256 + assert( (m.flags & MEM_Str)!=0 || db->mallocFailed );
1.257 + return (m.flags & MEM_Dyn)!=0 ? m.z : sqlite3DbStrDup(db, m.z);
1.258 }
1.259
1.260 /*
1.261 @@ -466,33 +492,6 @@
1.262
1.263 #if defined(SQLITE_TEST)
1.264 /*
1.265 -** Translate UTF-8 to UTF-8.
1.266 -**
1.267 -** This has the effect of making sure that the string is well-formed
1.268 -** UTF-8. Miscoded characters are removed.
1.269 -**
1.270 -** The translation is done in-place (since it is impossible for the
1.271 -** correct UTF-8 encoding to be longer than a malformed encoding).
1.272 -*/
1.273 -int sqlite3Utf8To8(unsigned char *zIn){
1.274 - unsigned char *zOut = zIn;
1.275 - unsigned char *zStart = zIn;
1.276 - int c;
1.277 -
1.278 - while(1){
1.279 - SQLITE_READ_UTF8(zIn, c);
1.280 - if( c==0 ) break;
1.281 - if( c!=0xfffd ){
1.282 - WRITE_UTF8(zOut, c);
1.283 - }
1.284 - }
1.285 - *zOut = 0;
1.286 - return zOut - zStart;
1.287 -}
1.288 -#endif
1.289 -
1.290 -#if defined(SQLITE_TEST)
1.291 -/*
1.292 ** This routine is called from the TCL test function "translate_selftest".
1.293 ** It checks that the primitives for serializing and deserializing
1.294 ** characters in each encoding are inverses of each other.
1.295 @@ -501,6 +500,7 @@
1.296 unsigned int i, t;
1.297 unsigned char zBuf[20];
1.298 unsigned char *z;
1.299 + unsigned char *zTerm;
1.300 int n;
1.301 unsigned int c;
1.302
1.303 @@ -509,8 +509,9 @@
1.304 WRITE_UTF8(z, i);
1.305 n = z-zBuf;
1.306 z[0] = 0;
1.307 + zTerm = z;
1.308 z = zBuf;
1.309 - SQLITE_READ_UTF8(z, c);
1.310 + c = sqlite3Utf8Read(z, zTerm, (const u8**)&z);
1.311 t = i;
1.312 if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD;
1.313 if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD;
