ds@1952: /* ds@1952: ** 2001 September 15 ds@1952: ** ds@1952: ** The author disclaims copyright to this source code. In place of ds@1952: ** a legal notice, here is a blessing: ds@1952: ** ds@1952: ** May you do good and not evil. ds@1952: ** May you find forgiveness for yourself and forgive others. ds@1952: ** May you share freely, never taking more than you give. ds@1952: ** ds@1952: ************************************************************************* ds@1952: ** An tokenizer for SQL ds@1952: ** ds@1952: ** This file contains C code that splits an SQL input string up into ds@1952: ** individual tokens and sends those tokens one-by-one over to the ds@1952: ** parser for analysis. ds@1952: ** ds@1952: ** $Id$ ds@1952: */ ds@1952: #include "sqliteInt.h" ds@1952: #include ds@1952: #include ds@1952: ds@1952: /* ds@1952: ** The charMap() macro maps alphabetic characters into their ds@1952: ** lower-case ASCII equivalent. On ASCII machines, this is just ds@1952: ** an upper-to-lower case map. On EBCDIC machines we also need ds@1952: ** to adjust the encoding. Only alphabetic characters and underscores ds@1952: ** need to be translated. ds@1952: */ ds@1952: #ifdef SQLITE_ASCII ds@1952: # define charMap(X) sqlite3UpperToLower[(unsigned char)X] ds@1952: #endif ds@1952: #ifdef SQLITE_EBCDIC ds@1952: # define charMap(X) ebcdicToAscii[(unsigned char)X] ds@1952: const unsigned char ebcdicToAscii[] = { ds@1952: /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ ds@1952: 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ ds@1952: 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ ds@1952: 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ ds@1952: 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ ds@1952: 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ ds@1952: 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ ds@1952: }; ds@1952: #endif ds@1952: ds@1952: /* ds@1952: ** The sqlite3KeywordCode function looks up an identifier to determine if ds@1952: ** it is a keyword. If it is a keyword, the token code of that keyword is ds@1952: ** returned. If the input is not a keyword, TK_ID is returned. ds@1952: ** ds@1952: ** The implementation of this routine was generated by a program, ds@1952: ** mkkeywordhash.h, located in the tool subdirectory of the distribution. ds@1952: ** The output of the mkkeywordhash.c program is written into a file ds@1952: ** named keywordhash.h and then included into this source file by ds@1952: ** the #include below. ds@1952: */ ds@1952: #include "keywordhash.h" ds@1952: ds@1952: ds@1952: /* ds@1952: ** If X is a character that can be used in an identifier then ds@1952: ** IdChar(X) will be true. Otherwise it is false. ds@1952: ** ds@1952: ** For ASCII, any character with the high-order bit set is ds@1952: ** allowed in an identifier. For 7-bit characters, ds@1952: ** sqlite3IsIdChar[X] must be 1. ds@1952: ** ds@1952: ** For EBCDIC, the rules are more complex but have the same ds@1952: ** end result. ds@1952: ** ds@1952: ** Ticket #1066. the SQL standard does not allow '$' in the ds@1952: ** middle of identfiers. But many SQL implementations do. ds@1952: ** SQLite will allow '$' in identifiers for compatibility. ds@1952: ** But the feature is undocumented. ds@1952: */ ds@1952: #ifdef SQLITE_ASCII ds@1952: const char sqlite3IsAsciiIdChar[] = { ds@1952: /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ ds@1952: 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ ds@1952: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ ds@1952: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ ds@1952: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ ds@1952: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ ds@1952: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ ds@1952: }; ds@1952: #define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20])) ds@1952: #endif ds@1952: #ifdef SQLITE_EBCDIC ds@1952: const char sqlite3IsEbcdicIdChar[] = { ds@1952: /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ ds@1952: 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ ds@1952: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ ds@1952: 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ ds@1952: 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ ds@1952: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ ds@1952: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ ds@1952: 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ ds@1952: 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ ds@1952: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ ds@1952: 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ ds@1952: 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ ds@1952: 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ ds@1952: }; ds@1952: #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) ds@1952: #endif ds@1952: ds@1952: ds@1952: /* ds@1952: ** Return the length of the token that begins at z[0]. ds@1952: ** Store the token type in *tokenType before returning. ds@1952: */ ds@1952: static int getToken(const unsigned char *z, int *tokenType){ ds@1952: int i, c; ds@1952: switch( *z ){ ds@1952: case ' ': case '\t': case '\n': case '\f': case '\r': { ds@1952: for(i=1; isspace(z[i]); i++){} ds@1952: *tokenType = TK_SPACE; ds@1952: return i; ds@1952: } ds@1952: case '-': { ds@1952: if( z[1]=='-' ){ ds@1952: for(i=2; (c=z[i])!=0 && c!='\n'; i++){} ds@1952: *tokenType = TK_COMMENT; ds@1952: return i; ds@1952: } ds@1952: *tokenType = TK_MINUS; ds@1952: return 1; ds@1952: } ds@1952: case '(': { ds@1952: *tokenType = TK_LP; ds@1952: return 1; ds@1952: } ds@1952: case ')': { ds@1952: *tokenType = TK_RP; ds@1952: return 1; ds@1952: } ds@1952: case ';': { ds@1952: *tokenType = TK_SEMI; ds@1952: return 1; ds@1952: } ds@1952: case '+': { ds@1952: *tokenType = TK_PLUS; ds@1952: return 1; ds@1952: } ds@1952: case '*': { ds@1952: *tokenType = TK_STAR; ds@1952: return 1; ds@1952: } ds@1952: case '/': { ds@1952: if( z[1]!='*' || z[2]==0 ){ ds@1952: *tokenType = TK_SLASH; ds@1952: return 1; ds@1952: } ds@1952: for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} ds@1952: if( c ) i++; ds@1952: *tokenType = TK_COMMENT; ds@1952: return i; ds@1952: } ds@1952: case '%': { ds@1952: *tokenType = TK_REM; ds@1952: return 1; ds@1952: } ds@1952: case '=': { ds@1952: *tokenType = TK_EQ; ds@1952: return 1 + (z[1]=='='); ds@1952: } ds@1952: case '<': { ds@1952: if( (c=z[1])=='=' ){ ds@1952: *tokenType = TK_LE; ds@1952: return 2; ds@1952: }else if( c=='>' ){ ds@1952: *tokenType = TK_NE; ds@1952: return 2; ds@1952: }else if( c=='<' ){ ds@1952: *tokenType = TK_LSHIFT; ds@1952: return 2; ds@1952: }else{ ds@1952: *tokenType = TK_LT; ds@1952: return 1; ds@1952: } ds@1952: } ds@1952: case '>': { ds@1952: if( (c=z[1])=='=' ){ ds@1952: *tokenType = TK_GE; ds@1952: return 2; ds@1952: }else if( c=='>' ){ ds@1952: *tokenType = TK_RSHIFT; ds@1952: return 2; ds@1952: }else{ ds@1952: *tokenType = TK_GT; ds@1952: return 1; ds@1952: } ds@1952: } ds@1952: case '!': { ds@1952: if( z[1]!='=' ){ ds@1952: *tokenType = TK_ILLEGAL; ds@1952: return 2; ds@1952: }else{ ds@1952: *tokenType = TK_NE; ds@1952: return 2; ds@1952: } ds@1952: } ds@1952: case '|': { ds@1952: if( z[1]!='|' ){ ds@1952: *tokenType = TK_BITOR; ds@1952: return 1; ds@1952: }else{ ds@1952: *tokenType = TK_CONCAT; ds@1952: return 2; ds@1952: } ds@1952: } ds@1952: case ',': { ds@1952: *tokenType = TK_COMMA; ds@1952: return 1; ds@1952: } ds@1952: case '&': { ds@1952: *tokenType = TK_BITAND; ds@1952: return 1; ds@1952: } ds@1952: case '~': { ds@1952: *tokenType = TK_BITNOT; ds@1952: return 1; ds@1952: } ds@1952: case '`': ds@1952: case '\'': ds@1952: case '"': { ds@1952: int delim = z[0]; ds@1952: for(i=1; (c=z[i])!=0; i++){ ds@1952: if( c==delim ){ ds@1952: if( z[i+1]==delim ){ ds@1952: i++; ds@1952: }else{ ds@1952: break; ds@1952: } ds@1952: } ds@1952: } ds@1952: if( c ){ ds@1952: *tokenType = TK_STRING; ds@1952: return i+1; ds@1952: }else{ ds@1952: *tokenType = TK_ILLEGAL; ds@1952: return i; ds@1952: } ds@1952: } ds@1952: case '.': { ds@1952: #ifndef SQLITE_OMIT_FLOATING_POINT ds@1952: if( !isdigit(z[1]) ) ds@1952: #endif ds@1952: { ds@1952: *tokenType = TK_DOT; ds@1952: return 1; ds@1952: } ds@1952: /* If the next character is a digit, this is a floating point ds@1952: ** number that begins with ".". Fall thru into the next case */ ds@1952: } ds@1952: case '0': case '1': case '2': case '3': case '4': ds@1952: case '5': case '6': case '7': case '8': case '9': { ds@1952: *tokenType = TK_INTEGER; ds@1952: for(i=0; isdigit(z[i]); i++){} ds@1952: #ifndef SQLITE_OMIT_FLOATING_POINT ds@1952: if( z[i]=='.' ){ ds@1952: i++; ds@1952: while( isdigit(z[i]) ){ i++; } ds@1952: *tokenType = TK_FLOAT; ds@1952: } ds@1952: if( (z[i]=='e' || z[i]=='E') && ds@1952: ( isdigit(z[i+1]) ds@1952: || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2])) ds@1952: ) ds@1952: ){ ds@1952: i += 2; ds@1952: while( isdigit(z[i]) ){ i++; } ds@1952: *tokenType = TK_FLOAT; ds@1952: } ds@1952: #endif ds@1952: while( IdChar(z[i]) ){ ds@1952: *tokenType = TK_ILLEGAL; ds@1952: i++; ds@1952: } ds@1952: return i; ds@1952: } ds@1952: case '[': { ds@1952: for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} ds@1952: *tokenType = TK_ID; ds@1952: return i; ds@1952: } ds@1952: case '?': { ds@1952: *tokenType = TK_VARIABLE; ds@1952: for(i=1; isdigit(z[i]); i++){} ds@1952: return i; ds@1952: } ds@1952: case '#': { ds@1952: for(i=1; isdigit(z[i]); i++){} ds@1952: if( i>1 ){ ds@1952: /* Parameters of the form #NNN (where NNN is a number) are used ds@1952: ** internally by sqlite3NestedParse. */ ds@1952: *tokenType = TK_REGISTER; ds@1952: return i; ds@1952: } ds@1952: /* Fall through into the next case if the '#' is not followed by ds@1952: ** a digit. Try to match #AAAA where AAAA is a parameter name. */ ds@1952: } ds@1952: #ifndef SQLITE_OMIT_TCL_VARIABLE ds@1952: case '$': ds@1952: #endif ds@1952: case '@': /* For compatibility with MS SQL Server */ ds@1952: case ':': { ds@1952: int n = 0; ds@1952: *tokenType = TK_VARIABLE; ds@1952: for(i=1; (c=z[i])!=0; i++){ ds@1952: if( IdChar(c) ){ ds@1952: n++; ds@1952: #ifndef SQLITE_OMIT_TCL_VARIABLE ds@1952: }else if( c=='(' && n>0 ){ ds@1952: do{ ds@1952: i++; ds@1952: }while( (c=z[i])!=0 && !isspace(c) && c!=')' ); ds@1952: if( c==')' ){ ds@1952: i++; ds@1952: }else{ ds@1952: *tokenType = TK_ILLEGAL; ds@1952: } ds@1952: break; ds@1952: }else if( c==':' && z[i+1]==':' ){ ds@1952: i++; ds@1952: #endif ds@1952: }else{ ds@1952: break; ds@1952: } ds@1952: } ds@1952: if( n==0 ) *tokenType = TK_ILLEGAL; ds@1952: return i; ds@1952: } ds@1952: #ifndef SQLITE_OMIT_BLOB_LITERAL ds@1952: case 'x': case 'X': { ds@1952: if( (c=z[1])=='\'' || c=='"' ){ ds@1952: int delim = c; ds@1952: *tokenType = TK_BLOB; ds@1952: for(i=2; (c=z[i])!=0; i++){ ds@1952: if( c==delim ){ ds@1952: if( i%2 ) *tokenType = TK_ILLEGAL; ds@1952: break; ds@1952: } ds@1952: if( !isxdigit(c) ){ ds@1952: *tokenType = TK_ILLEGAL; ds@1952: return i; ds@1952: } ds@1952: } ds@1952: if( c ) i++; ds@1952: return i; ds@1952: } ds@1952: /* Otherwise fall through to the next case */ ds@1952: } ds@1952: #endif ds@1952: default: { ds@1952: if( !IdChar(*z) ){ ds@1952: break; ds@1952: } ds@1952: for(i=1; IdChar(z[i]); i++){} ds@1952: *tokenType = keywordCode((char*)z, i); ds@1952: return i; ds@1952: } ds@1952: } ds@1952: *tokenType = TK_ILLEGAL; ds@1952: return 1; ds@1952: } ds@1952: int sqlite3GetToken(const unsigned char *z, int *tokenType){ ds@1952: return getToken(z, tokenType); ds@1952: } ds@1952: ds@1952: /* ds@1952: ** Run the parser on the given SQL string. The parser structure is ds@1952: ** passed in. An SQLITE_ status code is returned. If an error occurs ds@1952: ** and pzErrMsg!=NULL then an error message might be written into ds@1952: ** memory obtained from sqlite3_malloc() and *pzErrMsg made to point to that ds@1952: ** error message. Or maybe not. ds@1952: */ ds@1952: int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ ds@1952: int nErr = 0; ds@1952: int i; ds@1952: void *pEngine; ds@1952: int tokenType; ds@1952: int lastTokenParsed = -1; ds@1952: sqlite3 *db = pParse->db; ds@1952: ds@1952: if( db->activeVdbeCnt==0 ){ ds@1952: db->u1.isInterrupted = 0; ds@1952: } ds@1952: pParse->rc = SQLITE_OK; ds@1952: i = 0; ds@1952: pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3_malloc); ds@1952: if( pEngine==0 ){ ds@1952: db->mallocFailed = 1; ds@1952: return SQLITE_NOMEM; ds@1952: } ds@1952: assert( pParse->sLastToken.dyn==0 ); ds@1952: assert( pParse->pNewTable==0 ); ds@1952: assert( pParse->pNewTrigger==0 ); ds@1952: assert( pParse->nVar==0 ); ds@1952: assert( pParse->nVarExpr==0 ); ds@1952: assert( pParse->nVarExprAlloc==0 ); ds@1952: assert( pParse->apVarExpr==0 ); ds@1952: pParse->zTail = pParse->zSql = zSql; ds@1952: while( !db->mallocFailed && zSql[i]!=0 ){ ds@1952: assert( i>=0 ); ds@1952: pParse->sLastToken.z = (u8*)&zSql[i]; ds@1952: assert( pParse->sLastToken.dyn==0 ); ds@1952: pParse->sLastToken.n = getToken((unsigned char*)&zSql[i],&tokenType); ds@1952: i += pParse->sLastToken.n; ds@1952: if( i>SQLITE_MAX_SQL_LENGTH ){ ds@1952: pParse->rc = SQLITE_TOOBIG; ds@1952: break; ds@1952: } ds@1952: switch( tokenType ){ ds@1952: case TK_SPACE: ds@1952: case TK_COMMENT: { ds@1952: if( db->u1.isInterrupted ){ ds@1952: pParse->rc = SQLITE_INTERRUPT; ds@1952: sqlite3SetString(pzErrMsg, "interrupt", (char*)0); ds@1952: goto abort_parse; ds@1952: } ds@1952: break; ds@1952: } ds@1952: case TK_ILLEGAL: { ds@1952: if( pzErrMsg ){ ds@1952: sqlite3_free(*pzErrMsg); ds@1952: *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"", ds@1952: &pParse->sLastToken); ds@1952: } ds@1952: nErr++; ds@1952: goto abort_parse; ds@1952: } ds@1952: case TK_SEMI: { ds@1952: pParse->zTail = &zSql[i]; ds@1952: /* Fall thru into the default case */ ds@1952: } ds@1952: default: { ds@1952: sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); ds@1952: lastTokenParsed = tokenType; ds@1952: if( pParse->rc!=SQLITE_OK ){ ds@1952: goto abort_parse; ds@1952: } ds@1952: break; ds@1952: } ds@1952: } ds@1952: } ds@1952: abort_parse: ds@1952: if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ ds@1952: if( lastTokenParsed!=TK_SEMI ){ ds@1952: sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); ds@1952: pParse->zTail = &zSql[i]; ds@1952: } ds@1952: sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); ds@1952: } ds@1952: sqlite3ParserFree(pEngine, sqlite3_free); ds@1952: if( db->mallocFailed ){ ds@1952: pParse->rc = SQLITE_NOMEM; ds@1952: } ds@1952: if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ ds@1952: sqlite3SetString(&pParse->zErrMsg, sqlite3ErrStr(pParse->rc), (char*)0); ds@1952: } ds@1952: if( pParse->zErrMsg ){ ds@1952: if( pzErrMsg && *pzErrMsg==0 ){ ds@1952: *pzErrMsg = pParse->zErrMsg; ds@1952: }else{ ds@1952: sqlite3_free(pParse->zErrMsg); ds@1952: } ds@1952: pParse->zErrMsg = 0; ds@1952: if( !nErr ) nErr++; ds@1952: } ds@1952: if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ ds@1952: sqlite3VdbeDelete(pParse->pVdbe); ds@1952: pParse->pVdbe = 0; ds@1952: } ds@1952: #ifndef SQLITE_OMIT_SHARED_CACHE ds@1952: if( pParse->nested==0 ){ ds@1952: sqlite3_free(pParse->aTableLock); ds@1952: pParse->aTableLock = 0; ds@1952: pParse->nTableLock = 0; ds@1952: } ds@1952: #endif ds@1952: ds@1952: if( !IN_DECLARE_VTAB ){ ds@1952: /* If the pParse->declareVtab flag is set, do not delete any table ds@1952: ** structure built up in pParse->pNewTable. The calling code (see vtab.c) ds@1952: ** will take responsibility for freeing the Table structure. ds@1952: */ ds@1952: sqlite3DeleteTable(pParse->pNewTable); ds@1952: } ds@1952: ds@1952: sqlite3DeleteTrigger(pParse->pNewTrigger); ds@1952: sqlite3_free(pParse->apVarExpr); ds@1952: if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){ ds@1952: pParse->rc = SQLITE_ERROR; ds@1952: } ds@1952: return nErr; ds@1952: }