1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/extensions/sqlite/sqlite-source/tokenize.c Sun Jul 29 01:15:31 2007 +0000
1.3 @@ -0,0 +1,515 @@
1.4 +/*
1.5 +** 2001 September 15
1.6 +**
1.7 +** The author disclaims copyright to this source code. In place of
1.8 +** a legal notice, here is a blessing:
1.9 +**
1.10 +** May you do good and not evil.
1.11 +** May you find forgiveness for yourself and forgive others.
1.12 +** May you share freely, never taking more than you give.
1.13 +**
1.14 +*************************************************************************
1.15 +** An tokenizer for SQL
1.16 +**
1.17 +** This file contains C code that splits an SQL input string up into
1.18 +** individual tokens and sends those tokens one-by-one over to the
1.19 +** parser for analysis.
1.20 +**
1.21 +** $Id$
1.22 +*/
1.23 +#include "sqliteInt.h"
1.24 +#include "os.h"
1.25 +#include <ctype.h>
1.26 +#include <stdlib.h>
1.27 +
1.28 +/*
1.29 +** The charMap() macro maps alphabetic characters into their
1.30 +** lower-case ASCII equivalent. On ASCII machines, this is just
1.31 +** an upper-to-lower case map. On EBCDIC machines we also need
1.32 +** to adjust the encoding. Only alphabetic characters and underscores
1.33 +** need to be translated.
1.34 +*/
1.35 +#ifdef SQLITE_ASCII
1.36 +# define charMap(X) sqlite3UpperToLower[(unsigned char)X]
1.37 +#endif
1.38 +#ifdef SQLITE_EBCDIC
1.39 +# define charMap(X) ebcdicToAscii[(unsigned char)X]
1.40 +const unsigned char ebcdicToAscii[] = {
1.41 +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */
1.42 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
1.43 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
1.44 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
1.45 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */
1.46 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */
1.47 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */
1.48 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */
1.49 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */
1.50 + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */
1.51 + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */
1.52 + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */
1.53 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
1.54 + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */
1.55 + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */
1.56 + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */
1.57 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */
1.58 +};
1.59 +#endif
1.60 +
1.61 +/*
1.62 +** The sqlite3KeywordCode function looks up an identifier to determine if
1.63 +** it is a keyword. If it is a keyword, the token code of that keyword is
1.64 +** returned. If the input is not a keyword, TK_ID is returned.
1.65 +**
1.66 +** The implementation of this routine was generated by a program,
1.67 +** mkkeywordhash.h, located in the tool subdirectory of the distribution.
1.68 +** The output of the mkkeywordhash.c program is written into a file
1.69 +** named keywordhash.h and then included into this source file by
1.70 +** the #include below.
1.71 +*/
1.72 +#include "keywordhash.h"
1.73 +
1.74 +
1.75 +/*
1.76 +** If X is a character that can be used in an identifier then
1.77 +** IdChar(X) will be true. Otherwise it is false.
1.78 +**
1.79 +** For ASCII, any character with the high-order bit set is
1.80 +** allowed in an identifier. For 7-bit characters,
1.81 +** sqlite3IsIdChar[X] must be 1.
1.82 +**
1.83 +** For EBCDIC, the rules are more complex but have the same
1.84 +** end result.
1.85 +**
1.86 +** Ticket #1066. the SQL standard does not allow '$' in the
1.87 +** middle of identfiers. But many SQL implementations do.
1.88 +** SQLite will allow '$' in identifiers for compatibility.
1.89 +** But the feature is undocumented.
1.90 +*/
1.91 +#ifdef SQLITE_ASCII
1.92 +const char sqlite3IsIdChar[] = {
1.93 +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
1.94 + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
1.95 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
1.96 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1.97 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
1.98 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
1.99 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
1.100 +};
1.101 +#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsIdChar[c-0x20]))
1.102 +#endif
1.103 +#ifdef SQLITE_EBCDIC
1.104 +const char sqlite3IsIdChar[] = {
1.105 +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
1.106 + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */
1.107 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */
1.108 + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */
1.109 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */
1.110 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */
1.111 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */
1.112 + 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */
1.113 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
1.114 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */
1.115 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */
1.116 + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */
1.117 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */
1.118 +};
1.119 +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsIdChar[c-0x40]))
1.120 +#endif
1.121 +
1.122 +
1.123 +/*
1.124 +** Return the length of the token that begins at z[0].
1.125 +** Store the token type in *tokenType before returning.
1.126 +*/
1.127 +static int getToken(const unsigned char *z, int *tokenType){
1.128 + int i, c;
1.129 + switch( *z ){
1.130 + case ' ': case '\t': case '\n': case '\f': case '\r': {
1.131 + for(i=1; isspace(z[i]); i++){}
1.132 + *tokenType = TK_SPACE;
1.133 + return i;
1.134 + }
1.135 + case '-': {
1.136 + if( z[1]=='-' ){
1.137 + for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
1.138 + *tokenType = TK_COMMENT;
1.139 + return i;
1.140 + }
1.141 + *tokenType = TK_MINUS;
1.142 + return 1;
1.143 + }
1.144 + case '(': {
1.145 + *tokenType = TK_LP;
1.146 + return 1;
1.147 + }
1.148 + case ')': {
1.149 + *tokenType = TK_RP;
1.150 + return 1;
1.151 + }
1.152 + case ';': {
1.153 + *tokenType = TK_SEMI;
1.154 + return 1;
1.155 + }
1.156 + case '+': {
1.157 + *tokenType = TK_PLUS;
1.158 + return 1;
1.159 + }
1.160 + case '*': {
1.161 + *tokenType = TK_STAR;
1.162 + return 1;
1.163 + }
1.164 + case '/': {
1.165 + if( z[1]!='*' || z[2]==0 ){
1.166 + *tokenType = TK_SLASH;
1.167 + return 1;
1.168 + }
1.169 + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
1.170 + if( c ) i++;
1.171 + *tokenType = TK_COMMENT;
1.172 + return i;
1.173 + }
1.174 + case '%': {
1.175 + *tokenType = TK_REM;
1.176 + return 1;
1.177 + }
1.178 + case '=': {
1.179 + *tokenType = TK_EQ;
1.180 + return 1 + (z[1]=='=');
1.181 + }
1.182 + case '<': {
1.183 + if( (c=z[1])=='=' ){
1.184 + *tokenType = TK_LE;
1.185 + return 2;
1.186 + }else if( c=='>' ){
1.187 + *tokenType = TK_NE;
1.188 + return 2;
1.189 + }else if( c=='<' ){
1.190 + *tokenType = TK_LSHIFT;
1.191 + return 2;
1.192 + }else{
1.193 + *tokenType = TK_LT;
1.194 + return 1;
1.195 + }
1.196 + }
1.197 + case '>': {
1.198 + if( (c=z[1])=='=' ){
1.199 + *tokenType = TK_GE;
1.200 + return 2;
1.201 + }else if( c=='>' ){
1.202 + *tokenType = TK_RSHIFT;
1.203 + return 2;
1.204 + }else{
1.205 + *tokenType = TK_GT;
1.206 + return 1;
1.207 + }
1.208 + }
1.209 + case '!': {
1.210 + if( z[1]!='=' ){
1.211 + *tokenType = TK_ILLEGAL;
1.212 + return 2;
1.213 + }else{
1.214 + *tokenType = TK_NE;
1.215 + return 2;
1.216 + }
1.217 + }
1.218 + case '|': {
1.219 + if( z[1]!='|' ){
1.220 + *tokenType = TK_BITOR;
1.221 + return 1;
1.222 + }else{
1.223 + *tokenType = TK_CONCAT;
1.224 + return 2;
1.225 + }
1.226 + }
1.227 + case ',': {
1.228 + *tokenType = TK_COMMA;
1.229 + return 1;
1.230 + }
1.231 + case '&': {
1.232 + *tokenType = TK_BITAND;
1.233 + return 1;
1.234 + }
1.235 + case '~': {
1.236 + *tokenType = TK_BITNOT;
1.237 + return 1;
1.238 + }
1.239 + case '`':
1.240 + case '\'':
1.241 + case '"': {
1.242 + int delim = z[0];
1.243 + for(i=1; (c=z[i])!=0; i++){
1.244 + if( c==delim ){
1.245 + if( z[i+1]==delim ){
1.246 + i++;
1.247 + }else{
1.248 + break;
1.249 + }
1.250 + }
1.251 + }
1.252 + if( c ){
1.253 + *tokenType = TK_STRING;
1.254 + return i+1;
1.255 + }else{
1.256 + *tokenType = TK_ILLEGAL;
1.257 + return i;
1.258 + }
1.259 + }
1.260 + case '.': {
1.261 +#ifndef SQLITE_OMIT_FLOATING_POINT
1.262 + if( !isdigit(z[1]) )
1.263 +#endif
1.264 + {
1.265 + *tokenType = TK_DOT;
1.266 + return 1;
1.267 + }
1.268 + /* If the next character is a digit, this is a floating point
1.269 + ** number that begins with ".". Fall thru into the next case */
1.270 + }
1.271 + case '0': case '1': case '2': case '3': case '4':
1.272 + case '5': case '6': case '7': case '8': case '9': {
1.273 + *tokenType = TK_INTEGER;
1.274 + for(i=0; isdigit(z[i]); i++){}
1.275 +#ifndef SQLITE_OMIT_FLOATING_POINT
1.276 + if( z[i]=='.' ){
1.277 + i++;
1.278 + while( isdigit(z[i]) ){ i++; }
1.279 + *tokenType = TK_FLOAT;
1.280 + }
1.281 + if( (z[i]=='e' || z[i]=='E') &&
1.282 + ( isdigit(z[i+1])
1.283 + || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
1.284 + )
1.285 + ){
1.286 + i += 2;
1.287 + while( isdigit(z[i]) ){ i++; }
1.288 + *tokenType = TK_FLOAT;
1.289 + }
1.290 +#endif
1.291 + while( IdChar(z[i]) ){
1.292 + *tokenType = TK_ILLEGAL;
1.293 + i++;
1.294 + }
1.295 + return i;
1.296 + }
1.297 + case '[': {
1.298 + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
1.299 + *tokenType = TK_ID;
1.300 + return i;
1.301 + }
1.302 + case '?': {
1.303 + *tokenType = TK_VARIABLE;
1.304 + for(i=1; isdigit(z[i]); i++){}
1.305 + return i;
1.306 + }
1.307 + case '#': {
1.308 + for(i=1; isdigit(z[i]); i++){}
1.309 + if( i>1 ){
1.310 + /* Parameters of the form #NNN (where NNN is a number) are used
1.311 + ** internally by sqlite3NestedParse. */
1.312 + *tokenType = TK_REGISTER;
1.313 + return i;
1.314 + }
1.315 + /* Fall through into the next case if the '#' is not followed by
1.316 + ** a digit. Try to match #AAAA where AAAA is a parameter name. */
1.317 + }
1.318 +#ifndef SQLITE_OMIT_TCL_VARIABLE
1.319 + case '$':
1.320 +#endif
1.321 + case '@': /* For compatibility with MS SQL Server */
1.322 + case ':': {
1.323 + int n = 0;
1.324 + *tokenType = TK_VARIABLE;
1.325 + for(i=1; (c=z[i])!=0; i++){
1.326 + if( IdChar(c) ){
1.327 + n++;
1.328 +#ifndef SQLITE_OMIT_TCL_VARIABLE
1.329 + }else if( c=='(' && n>0 ){
1.330 + do{
1.331 + i++;
1.332 + }while( (c=z[i])!=0 && !isspace(c) && c!=')' );
1.333 + if( c==')' ){
1.334 + i++;
1.335 + }else{
1.336 + *tokenType = TK_ILLEGAL;
1.337 + }
1.338 + break;
1.339 + }else if( c==':' && z[i+1]==':' ){
1.340 + i++;
1.341 +#endif
1.342 + }else{
1.343 + break;
1.344 + }
1.345 + }
1.346 + if( n==0 ) *tokenType = TK_ILLEGAL;
1.347 + return i;
1.348 + }
1.349 +#ifndef SQLITE_OMIT_BLOB_LITERAL
1.350 + case 'x': case 'X': {
1.351 + if( (c=z[1])=='\'' || c=='"' ){
1.352 + int delim = c;
1.353 + *tokenType = TK_BLOB;
1.354 + for(i=2; (c=z[i])!=0; i++){
1.355 + if( c==delim ){
1.356 + if( i%2 ) *tokenType = TK_ILLEGAL;
1.357 + break;
1.358 + }
1.359 + if( !isxdigit(c) ){
1.360 + *tokenType = TK_ILLEGAL;
1.361 + return i;
1.362 + }
1.363 + }
1.364 + if( c ) i++;
1.365 + return i;
1.366 + }
1.367 + /* Otherwise fall through to the next case */
1.368 + }
1.369 +#endif
1.370 + default: {
1.371 + if( !IdChar(*z) ){
1.372 + break;
1.373 + }
1.374 + for(i=1; IdChar(z[i]); i++){}
1.375 + *tokenType = keywordCode((char*)z, i);
1.376 + return i;
1.377 + }
1.378 + }
1.379 + *tokenType = TK_ILLEGAL;
1.380 + return 1;
1.381 +}
1.382 +int sqlite3GetToken(const unsigned char *z, int *tokenType){
1.383 + return getToken(z, tokenType);
1.384 +}
1.385 +
1.386 +/*
1.387 +** The interface to the LEMON-generated parser
1.388 +*/
1.389 +void *sqlite3ParserAlloc(void*(*)(size_t));
1.390 +void sqlite3ParserFree(void*, void(*)(void*));
1.391 +void sqlite3Parser(void*, int, Token, Parse*);
1.392 +
1.393 +/*
1.394 +** Run the parser on the given SQL string. The parser structure is
1.395 +** passed in. An SQLITE_ status code is returned. If an error occurs
1.396 +** and pzErrMsg!=NULL then an error message might be written into
1.397 +** memory obtained from malloc() and *pzErrMsg made to point to that
1.398 +** error message. Or maybe not.
1.399 +*/
1.400 +int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
1.401 + int nErr = 0;
1.402 + int i;
1.403 + void *pEngine;
1.404 + int tokenType;
1.405 + int lastTokenParsed = -1;
1.406 + sqlite3 *db = pParse->db;
1.407 +
1.408 + if( db->activeVdbeCnt==0 ){
1.409 + db->u1.isInterrupted = 0;
1.410 + }
1.411 + pParse->rc = SQLITE_OK;
1.412 + i = 0;
1.413 + pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3MallocX);
1.414 + if( pEngine==0 ){
1.415 + return SQLITE_NOMEM;
1.416 + }
1.417 + assert( pParse->sLastToken.dyn==0 );
1.418 + assert( pParse->pNewTable==0 );
1.419 + assert( pParse->pNewTrigger==0 );
1.420 + assert( pParse->nVar==0 );
1.421 + assert( pParse->nVarExpr==0 );
1.422 + assert( pParse->nVarExprAlloc==0 );
1.423 + assert( pParse->apVarExpr==0 );
1.424 + pParse->zTail = pParse->zSql = zSql;
1.425 + while( !sqlite3MallocFailed() && zSql[i]!=0 ){
1.426 + assert( i>=0 );
1.427 + pParse->sLastToken.z = (u8*)&zSql[i];
1.428 + assert( pParse->sLastToken.dyn==0 );
1.429 + pParse->sLastToken.n = getToken((unsigned char*)&zSql[i],&tokenType);
1.430 + i += pParse->sLastToken.n;
1.431 + if( i>SQLITE_MAX_SQL_LENGTH ){
1.432 + pParse->rc = SQLITE_TOOBIG;
1.433 + break;
1.434 + }
1.435 + switch( tokenType ){
1.436 + case TK_SPACE:
1.437 + case TK_COMMENT: {
1.438 + if( db->u1.isInterrupted ){
1.439 + pParse->rc = SQLITE_INTERRUPT;
1.440 + sqlite3SetString(pzErrMsg, "interrupt", (char*)0);
1.441 + goto abort_parse;
1.442 + }
1.443 + break;
1.444 + }
1.445 + case TK_ILLEGAL: {
1.446 + if( pzErrMsg ){
1.447 + sqliteFree(*pzErrMsg);
1.448 + *pzErrMsg = sqlite3MPrintf("unrecognized token: \"%T\"",
1.449 + &pParse->sLastToken);
1.450 + }
1.451 + nErr++;
1.452 + goto abort_parse;
1.453 + }
1.454 + case TK_SEMI: {
1.455 + pParse->zTail = &zSql[i];
1.456 + /* Fall thru into the default case */
1.457 + }
1.458 + default: {
1.459 + sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
1.460 + lastTokenParsed = tokenType;
1.461 + if( pParse->rc!=SQLITE_OK ){
1.462 + goto abort_parse;
1.463 + }
1.464 + break;
1.465 + }
1.466 + }
1.467 + }
1.468 +abort_parse:
1.469 + if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){
1.470 + if( lastTokenParsed!=TK_SEMI ){
1.471 + sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
1.472 + pParse->zTail = &zSql[i];
1.473 + }
1.474 + sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
1.475 + }
1.476 + sqlite3ParserFree(pEngine, sqlite3FreeX);
1.477 + if( sqlite3MallocFailed() ){
1.478 + pParse->rc = SQLITE_NOMEM;
1.479 + }
1.480 + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
1.481 + sqlite3SetString(&pParse->zErrMsg, sqlite3ErrStr(pParse->rc), (char*)0);
1.482 + }
1.483 + if( pParse->zErrMsg ){
1.484 + if( pzErrMsg && *pzErrMsg==0 ){
1.485 + *pzErrMsg = pParse->zErrMsg;
1.486 + }else{
1.487 + sqliteFree(pParse->zErrMsg);
1.488 + }
1.489 + pParse->zErrMsg = 0;
1.490 + if( !nErr ) nErr++;
1.491 + }
1.492 + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){
1.493 + sqlite3VdbeDelete(pParse->pVdbe);
1.494 + pParse->pVdbe = 0;
1.495 + }
1.496 +#ifndef SQLITE_OMIT_SHARED_CACHE
1.497 + if( pParse->nested==0 ){
1.498 + sqliteFree(pParse->aTableLock);
1.499 + pParse->aTableLock = 0;
1.500 + pParse->nTableLock = 0;
1.501 + }
1.502 +#endif
1.503 +
1.504 + if( !IN_DECLARE_VTAB ){
1.505 + /* If the pParse->declareVtab flag is set, do not delete any table
1.506 + ** structure built up in pParse->pNewTable. The calling code (see vtab.c)
1.507 + ** will take responsibility for freeing the Table structure.
1.508 + */
1.509 + sqlite3DeleteTable(pParse->pNewTable);
1.510 + }
1.511 +
1.512 + sqlite3DeleteTrigger(pParse->pNewTrigger);
1.513 + sqliteFree(pParse->apVarExpr);
1.514 + if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){
1.515 + pParse->rc = SQLITE_ERROR;
1.516 + }
1.517 + return nErr;
1.518 +}
