aboutsummaryrefslogtreecommitdiff
path: root/src/tokenize.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenize.c')
-rw-r--r--src/tokenize.c152
1 files changed, 71 insertions, 81 deletions
diff --git a/src/tokenize.c b/src/tokenize.c
index 8467c0ffe..f4d013dee 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -27,8 +27,8 @@
** all of them need to be used within the switch.
*/
#define CC_X 0 /* The letter 'x', or start of BLOB literal */
-#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */
-#define CC_ID 2 /* unicode characters usable in IDs */
+#define CC_KYWD0 1 /* First letter of a keyword */
+#define CC_KYWD 2 /* Alphabetics or '_'. Usable in a keyword */
#define CC_DIGIT 3 /* Digits */
#define CC_DOLLAR 4 /* '$' */
#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */
@@ -53,47 +53,49 @@
#define CC_AND 24 /* '&' */
#define CC_TILDA 25 /* '~' */
#define CC_DOT 26 /* '.' */
-#define CC_ILLEGAL 27 /* Illegal character */
-#define CC_NUL 28 /* 0x00 */
+#define CC_ID 27 /* unicode characters usable in IDs */
+#define CC_ILLEGAL 28 /* Illegal character */
+#define CC_NUL 29 /* 0x00 */
+#define CC_BOM 30 /* First byte of UTF8 BOM: 0xEF 0xBB 0xBF */
static const unsigned char aiClass[] = {
#ifdef SQLITE_ASCII
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
-/* 0x */ 28, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27,
-/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* 0x */ 29, 28, 28, 28, 28, 28, 28, 28, 28, 7, 7, 28, 7, 7, 28, 28,
+/* 1x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16,
/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6,
/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1,
+/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 9, 28, 28, 28, 2,
/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27,
-/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
+/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 28, 10, 28, 25, 28,
+/* 8x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* 9x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* Ax */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* Cx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* Dx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* Ex */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 30,
+/* Fx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27
#endif
#ifdef SQLITE_EBCDIC
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
-/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27,
-/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
-/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
-/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
-/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 12, 17, 20, 10,
-/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27,
-/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 6,
-/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8,
-/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
-/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
-/* Ax */ 27, 25, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27,
-/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27,
-/* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
-/* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
-/* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27,
-/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27,
+/* 0x */ 29, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 7, 7, 28, 28,
+/* 1x */ 28, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
+/* 2x */ 28, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
+/* 3x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
+/* 4x */ 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 26, 12, 17, 20, 10,
+/* 5x */ 24, 28, 28, 28, 28, 28, 28, 28, 28, 28, 15, 4, 21, 18, 19, 28,
+/* 6x */ 11, 16, 28, 28, 28, 28, 28, 28, 28, 28, 28, 23, 22, 2, 13, 6,
+/* 7x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 8, 5, 5, 5, 8, 14, 8,
+/* 8x */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28,
+/* 9x */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28,
+/* Ax */ 28, 25, 1, 1, 1, 1, 1, 0, 2, 2, 28, 28, 28, 28, 28, 28,
+/* Bx */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 9, 28, 28, 28, 28, 28,
+/* Cx */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28,
+/* Dx */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28,
+/* Ex */ 28, 28, 1, 1, 1, 1, 1, 0, 2, 2, 28, 28, 28, 28, 28, 28,
+/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 28, 28, 28, 28, 28, 28,
#endif
};
@@ -288,6 +290,9 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
*tokenType = TK_SPACE; /* IMP: R-22934-25134 */
return i;
+ }else if( z[1]=='>' ){
+ *tokenType = TK_PTR;
+ return 2 + (z[2]=='>');
}
*tokenType = TK_MINUS;
return 1;
@@ -422,12 +427,13 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
}
/* If the next character is a digit, this is a floating point
** number that begins with ".". Fall thru into the next case */
+ /* no break */ deliberate_fall_through
}
case CC_DIGIT: {
testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' );
testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' );
testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' );
- testcase( z[0]=='9' );
+ testcase( z[0]=='9' ); testcase( z[0]=='.' );
*tokenType = TK_INTEGER;
#ifndef SQLITE_OMIT_HEX_INTEGER
if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
@@ -498,8 +504,9 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
if( n==0 ) *tokenType = TK_ILLEGAL;
return i;
}
- case CC_KYWD: {
- for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+ case CC_KYWD0: {
+ if( aiClass[z[1]]>CC_KYWD ){ i = 1; break; }
+ for(i=2; aiClass[z[i]]<=CC_KYWD; i++){}
if( IdChar(z[i]) ){
/* This token started out using characters that can appear in keywords,
** but z[i] is a character not allowed within keywords, so this must
@@ -526,11 +533,21 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
#endif
/* If it is not a BLOB literal, then it must be an ID, since no
** SQL keywords start with the letter 'x'. Fall through */
+ /* no break */ deliberate_fall_through
}
+ case CC_KYWD:
case CC_ID: {
i = 1;
break;
}
+ case CC_BOM: {
+ if( z[1]==0xbb && z[2]==0xbf ){
+ *tokenType = TK_SPACE;
+ return 3;
+ }
+ i = 1;
+ break;
+ }
case CC_NUL: {
*tokenType = TK_ILLEGAL;
return 0;
@@ -546,13 +563,9 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
}
/*
-** Run the parser on the given SQL string. The parser structure is
-** passed in. An SQLITE_ status code is returned. If an error occurs
-** then an and attempt is made to write an error message into
-** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
-** error message.
+** Run the parser on the given SQL string.
*/
-int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
+int sqlite3RunParser(Parse *pParse, const char *zSql){
int nErr = 0; /* Number of errors encountered */
void *pEngine; /* The LEMON-generated LALR(1) parser */
int n = 0; /* Length of the next token token */
@@ -560,6 +573,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
int lastTokenParsed = -1; /* type of the previous token */
sqlite3 *db = pParse->db; /* The database connection */
int mxSqlLen; /* Max length of an SQL string */
+ Parse *pParentParse = 0; /* Outer parse context, if any */
#ifdef sqlite3Parser_ENGINEALWAYSONSTACK
yyParser sEngine; /* Space to hold the Lemon-generated Parser object */
#endif
@@ -572,7 +586,6 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
}
pParse->rc = SQLITE_OK;
pParse->zTail = zSql;
- assert( pzErrMsg!=0 );
#ifdef SQLITE_DEBUG
if( db->flags & SQLITE_ParserTrace ){
printf("parser: [[[%s]]]\n", zSql);
@@ -595,13 +608,14 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
assert( pParse->pNewTrigger==0 );
assert( pParse->nVar==0 );
assert( pParse->pVList==0 );
- pParse->pParentParse = db->pParse;
+ pParentParse = db->pParse;
db->pParse = pParse;
while( 1 ){
n = sqlite3GetToken((u8*)zSql, &tokenType);
mxSqlLen -= n;
if( mxSqlLen<0 ){
pParse->rc = SQLITE_TOOBIG;
+ pParse->nErr++;
break;
}
#ifndef SQLITE_OMIT_WINDOWFUNC
@@ -615,6 +629,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
#endif /* SQLITE_OMIT_WINDOWFUNC */
if( AtomicLoad(&db->u1.isInterrupted) ){
pParse->rc = SQLITE_INTERRUPT;
+ pParse->nErr++;
break;
}
if( tokenType==TK_SPACE ){
@@ -644,7 +659,10 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed);
#endif /* SQLITE_OMIT_WINDOWFUNC */
}else{
- sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
+ Token x;
+ x.z = zSql;
+ x.n = n;
+ sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &x);
break;
}
}
@@ -672,58 +690,30 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
if( db->mallocFailed ){
pParse->rc = SQLITE_NOMEM_BKPT;
}
- if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
- pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc));
- }
- assert( pzErrMsg!=0 );
- if( pParse->zErrMsg ){
- *pzErrMsg = pParse->zErrMsg;
- sqlite3_log(pParse->rc, "%s in \"%s\"",
- *pzErrMsg, pParse->zTail);
- pParse->zErrMsg = 0;
+ if( pParse->zErrMsg || (pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE) ){
+ if( pParse->zErrMsg==0 ){
+ pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc));
+ }
+ sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail);
nErr++;
}
pParse->zTail = zSql;
- if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){
- sqlite3VdbeDelete(pParse->pVdbe);
- pParse->pVdbe = 0;
- }
-#ifndef SQLITE_OMIT_SHARED_CACHE
- if( pParse->nested==0 ){
- sqlite3DbFree(db, pParse->aTableLock);
- pParse->aTableLock = 0;
- pParse->nTableLock = 0;
- }
-#endif
#ifndef SQLITE_OMIT_VIRTUALTABLE
sqlite3_free(pParse->apVtabLock);
#endif
- if( !IN_SPECIAL_PARSE ){
+ if( pParse->pNewTable && !IN_SPECIAL_PARSE ){
/* If the pParse->declareVtab flag is set, do not delete any table
** structure built up in pParse->pNewTable. The calling code (see vtab.c)
** will take responsibility for freeing the Table structure.
*/
sqlite3DeleteTable(db, pParse->pNewTable);
}
- if( !IN_RENAME_OBJECT ){
+ if( pParse->pNewTrigger && !IN_RENAME_OBJECT ){
sqlite3DeleteTrigger(db, pParse->pNewTrigger);
}
-
- if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree);
- sqlite3DbFree(db, pParse->pVList);
- while( pParse->pAinc ){
- AutoincInfo *p = pParse->pAinc;
- pParse->pAinc = p->pNext;
- sqlite3DbFreeNN(db, p);
- }
- while( pParse->pZombieTab ){
- Table *p = pParse->pZombieTab;
- pParse->pZombieTab = p->pNextZombie;
- sqlite3DeleteTable(db, p);
- }
- db->pParse = pParse->pParentParse;
- pParse->pParentParse = 0;
+ if( pParse->pVList ) sqlite3DbNNFreeNN(db, pParse->pVList);
+ db->pParse = pParentParse;
assert( nErr==0 || pParse->rc!=SQLITE_OK );
return nErr;
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-18 02:06:56 +0000