aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/build.c6
-rw-r--r--src/callback.c19
-rw-r--r--src/ctime.c3
-rw-r--r--src/expr.c8
-rw-r--r--src/hash.c56
-rw-r--r--src/loadext.c8
-rw-r--r--src/prepare.c288
-rw-r--r--src/sqlite.h.in21
-rw-r--r--src/sqlite3ext.h5
-rw-r--r--src/sqliteInt.h23
-rw-r--r--src/test1.c100
-rw-r--r--src/test_config.c6
-rw-r--r--src/tokenize.c67
-rw-r--r--src/vdbeInt.h3
-rw-r--r--src/vdbeapi.c10
-rw-r--r--src/vdbeaux.c15
16 files changed, 632 insertions, 6 deletions
diff --git a/src/build.c b/src/build.c
index b4041389b..5a5bea50f 100644
--- a/src/build.c
+++ b/src/build.c
@@ -633,6 +633,12 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
/* Delete the Table structure itself.
*/
+#ifdef SQLITE_ENABLE_NORMALIZE
+ if( pTable->pColHash ){
+ sqlite3HashClear(pTable->pColHash);
+ sqlite3_free(pTable->pColHash);
+ }
+#endif
sqlite3DeleteColumnNames(db, pTable);
sqlite3DbFree(db, pTable->zName);
sqlite3DbFree(db, pTable->zColAff);
diff --git a/src/callback.c b/src/callback.c
index a629b6825..faf6d520c 100644
--- a/src/callback.c
+++ b/src/callback.c
@@ -295,6 +295,21 @@ static FuncDef *functionSearch(
}
return 0;
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+FuncDef *sqlite3FunctionSearchN(
+ int h, /* Hash of the name */
+ const char *zFunc, /* Name of function */
+ int nFunc /* Length of the name */
+){
+ FuncDef *p;
+ for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
+ if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){
+ return p;
+ }
+ }
+ return 0;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
/*
** Insert a new FuncDef into a FuncDefHash hash table.
@@ -308,7 +323,7 @@ void sqlite3InsertBuiltinFuncs(
FuncDef *pOther;
const char *zName = aDef[i].zName;
int nName = sqlite3Strlen30(zName);
- int h = (zName[0] + nName) % SQLITE_FUNC_HASH_SZ;
+ int h = SQLITE_FUNC_HASH(zName[0], nName);
assert( zName[0]>='a' && zName[0]<='z' );
pOther = functionSearch(h, zName);
if( pOther ){
@@ -387,7 +402,7 @@ FuncDef *sqlite3FindFunction(
*/
if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
bestScore = 0;
- h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ;
+ h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
p = functionSearch(h, zName);
while( p ){
int score = matchQuality(p, nArg, enc);
diff --git a/src/ctime.c b/src/ctime.c
index 5bc1e5c74..27fc4fe6c 100644
--- a/src/ctime.c
+++ b/src/ctime.c
@@ -268,6 +268,9 @@ static const char * const sqlite3azCompileOpt[] = {
#if SQLITE_ENABLE_MULTIPLEX
"ENABLE_MULTIPLEX",
#endif
+#if SQLITE_ENABLE_NORMALIZE
+ "ENABLE_NORMALIZE",
+#endif
#if SQLITE_ENABLE_NULL_TRIM
"ENABLE_NULL_TRIM",
#endif
diff --git a/src/expr.c b/src/expr.c
index b2854f9a5..9c0ce1f87 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -2149,6 +2149,14 @@ int sqlite3IsRowid(const char *z){
if( sqlite3StrICmp(z, "OID")==0 ) return 1;
return 0;
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+int sqlite3IsRowidN(const char *z, int n){
+ if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1;
+ if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1;
+ if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1;
+ return 0;
+}
+#endif
/*
** pX is the RHS of an IN operator. If pX is a SELECT statement
diff --git a/src/hash.c b/src/hash.c
index 79bb85ac0..fba9dc9f8 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -64,6 +64,20 @@ static unsigned int strHash(const char *z){
}
return h;
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+static unsigned int strHashN(const char *z, int n){
+ unsigned int h = 0;
+ int i;
+ for(i=0; i<n; i++){
+ /* Knuth multiplicative hashing. (Sorting & Searching, p. 510).
+ ** 0x9e3779b1 is 2654435761 which is the closest prime number to
+ ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
+ h += sqlite3UpperToLower[z[i]];
+ h *= 0x9e3779b1;
+ }
+ return h;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
/* Link pNew element into the hash table pH. If pEntry!=0 then also
@@ -175,6 +189,40 @@ static HashElem *findElementWithHash(
}
return &nullElement;
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+static HashElem *findElementWithHashN(
+ const Hash *pH, /* The pH to be searched */
+ const char *pKey, /* The key we are searching for */
+ int nKey, /* Number of key bytes to use */
+ unsigned int *pHash /* Write the hash value here */
+){
+ HashElem *elem; /* Used to loop thru the element list */
+ int count; /* Number of elements left to test */
+ unsigned int h; /* The computed hash */
+ static HashElem nullElement = { 0, 0, 0, 0 };
+
+ if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/
+ struct _ht *pEntry;
+ h = strHashN(pKey, nKey) % pH->htsize;
+ pEntry = &pH->ht[h];
+ elem = pEntry->chain;
+ count = pEntry->count;
+ }else{
+ h = 0;
+ elem = pH->first;
+ count = pH->count;
+ }
+ if( pHash ) *pHash = h;
+ while( count-- ){
+ assert( elem!=0 );
+ if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){
+ return elem;
+ }
+ elem = elem->next;
+ }
+ return &nullElement;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
@@ -219,6 +267,14 @@ void *sqlite3HashFind(const Hash *pH, const char *pKey){
assert( pKey!=0 );
return findElementWithHash(pH, pKey, 0)->data;
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){
+ assert( pH!=0 );
+ assert( pKey!=0 );
+ assert( nKey>=0 );
+ return findElementWithHashN(pH, pKey, nKey, 0)->data;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
/* Insert an element into the hash table pH. The key is pKey
** and the data is "data".
diff --git a/src/loadext.c b/src/loadext.c
index 72bfd5c51..710227d7c 100644
--- a/src/loadext.c
+++ b/src/loadext.c
@@ -451,7 +451,13 @@ static const sqlite3_api_routines sqlite3Apis = {
sqlite3_str_length,
sqlite3_str_value,
/* Version 3.25.0 and later */
- sqlite3_create_window_function
+ sqlite3_create_window_function,
+ /* Version 3.26.0 and later */
+#ifdef SQLITE_ENABLE_NORMALIZE
+ sqlite3_normalized_sql
+#else
+ 0
+#endif
};
/*
diff --git a/src/prepare.c b/src/prepare.c
index 602e4dc49..6d2b80413 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -709,6 +709,294 @@ static int sqlite3LockAndPrepare(
return rc;
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Checks if the specified token is a table, column, or function name,
+** based on the databases associated with the statement being prepared.
+** If the function fails, zero is returned and pRc is filled with the
+** error code.
+*/
+static int shouldTreatAsIdentifier(
+ sqlite3 *db, /* Database handle. */
+ const char *zToken, /* Pointer to start of token to be checked */
+ int nToken, /* Length of token to be checked */
+ int *pRc /* Pointer to error code upon failure */
+){
+ int bFound = 0; /* Non-zero if token is an identifier name. */
+ int i, j; /* Database and column loop indexes. */
+ Schema *pSchema; /* Schema for current database. */
+ Hash *pHash; /* Hash table of tables for current database. */
+ HashElem *e; /* Hash element for hash table iteration. */
+ Table *pTab; /* Database table for columns being checked. */
+
+ if( sqlite3IsRowidN(zToken, nToken) ){
+ return 1;
+ }
+ if( nToken>0 ){
+ int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
+ if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1;
+ }
+ assert( db!=0 );
+ sqlite3_mutex_enter(db->mutex);
+ sqlite3BtreeEnterAll(db);
+ for(i=0; i<db->nDb; i++){
+ pHash = &db->aFunc;
+ if( sqlite3HashFindN(pHash, zToken, nToken) ){
+ bFound = 1;
+ break;
+ }
+ pSchema = db->aDb[i].pSchema;
+ if( pSchema==0 ) continue;
+ pHash = &pSchema->tblHash;
+ if( sqlite3HashFindN(pHash, zToken, nToken) ){
+ bFound = 1;
+ break;
+ }
+ for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
+ pTab = sqliteHashData(e);
+ if( pTab==0 ) continue;
+ pHash = pTab->pColHash;
+ if( pHash==0 ){
+ pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash));
+ if( pHash ){
+ sqlite3HashInit(pHash);
+ for(j=0; j<pTab->nCol; j++){
+ Column *pCol = &pTab->aCol[j];
+ sqlite3HashInsert(pHash, pCol->zName, pCol);
+ }
+ }else{
+ *pRc = SQLITE_NOMEM_BKPT;
+ bFound = 0;
+ goto done;
+ }
+ }
+ if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){
+ bFound = 1;
+ goto done;
+ }
+ }
+ }
+done:
+ sqlite3BtreeLeaveAll(db);
+ sqlite3_mutex_leave(db->mutex);
+ return bFound;
+}
+
+/*
+** Attempt to estimate the final output buffer size needed for the fully
+** normalized version of the specified SQL string. This should take into
+** account any potential expansion that could occur (e.g. via IN clauses
+** being expanded, etc). This size returned is the total number of bytes
+** including the NUL terminator.
+*/
+static int estimateNormalizedSize(
+ const char *zSql, /* The original SQL string */
+ int nSql, /* Length of original SQL string */
+ u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */
+){
+ int nOut = nSql + 4;
+ const char *z = zSql;
+ while( nOut<nSql*5 ){
+ while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
+ if( z[0]==0 ) break;
+ z++;
+ if( z[0]!='N' && z[0]!='n' ) break;
+ z++;
+ while( sqlite3Isspace(z[0]) ){ z++; }
+ if( z[0]!='(' ) break;
+ z++;
+ nOut += 5; /* ?,?,? */
+ }
+ return nOut;
+}
+
+/*
+** Copy the current token into the output buffer while dealing with quoted
+** identifiers. By default, all letters will be converted into lowercase.
+** If the bUpper flag is set, uppercase will be used. The piOut argument
+** will be used to update the target index into the output string.
+*/
+static void copyNormalizedToken(
+ const char *zSql, /* The original SQL string */
+ int iIn, /* Current index into the original SQL string */
+ int nToken, /* Number of bytes in the current token */
+ int tokenFlags, /* Flags returned by the tokenizer */
+ char *zOut, /* The output string */
+ int *piOut /* Pointer to target index into the output string */
+){
+ int bQuoted = tokenFlags & SQLITE_TOKEN_QUOTED;
+ int bKeyword = tokenFlags & SQLITE_TOKEN_KEYWORD;
+ int j = *piOut, k = 0;
+ for(; k<nToken; k++){
+ if( bQuoted ){
+ if( k==0 && iIn>0 ){
+ zOut[j++] = '"';
+ continue;
+ }else if( k==nToken-1 ){
+ zOut[j++] = '"';
+ continue;
+ }
+ }
+ if( bKeyword ){
+ zOut[j++] = sqlite3Toupper(zSql[iIn+k]);
+ }else{
+ zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
+ }
+ }
+ *piOut = j;
+}
+
+/*
+** Perform normalization of the SQL contained in the prepared statement and
+** store the result in the zNormSql field. The schema for the associated
+** databases are consulted while performing the normalization in order to
+** determine if a token appears to be an identifier. All identifiers are
+** left intact in the normalized SQL and all literals are replaced with a
+** single '?'.
+*/
+void sqlite3Normalize(
+ Vdbe *pVdbe, /* VM being reprepared */
+ const char *zSql, /* The original SQL string */
+ int nSql, /* Size of the input string in bytes */
+ u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */
+){
+ sqlite3 *db; /* Database handle. */
+ char *z; /* The output string */
+ int nZ; /* Size of the output string in bytes */
+ int i; /* Next character to read from zSql[] */
+ int j; /* Next character to fill in on z[] */
+ int tokenType = 0; /* Type of the next token */
+ int prevTokenType = 0; /* Type of the previous token, except spaces */
+ int n; /* Size of the next token */
+ int nParen = 0; /* Nesting level of parenthesis */
+ Hash inHash; /* Table of parenthesis levels to output index. */
+
+ db = sqlite3VdbeDb(pVdbe);
+ assert( db!=0 );
+ assert( pVdbe->zNormSql==0 );
+ if( zSql==0 ) return;
+ nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
+ z = sqlite3DbMallocRawNN(db, nZ);
+ if( z==0 ) return;
+ sqlite3HashInit(&inHash);
+ for(i=j=0; i<nSql && zSql[i]; i+=n){
+ int flags = 0;
+ if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
+ n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
+ switch( tokenType ){
+ case TK_SPACE: {
+ break;
+ }
+ case TK_ILLEGAL: {
+ sqlite3DbFree(db, z);
+ sqlite3HashClear(&inHash);
+ return;
+ }
+ case TK_STRING:
+ case TK_INTEGER:
+ case TK_FLOAT:
+ case TK_VARIABLE:
+ case TK_BLOB: {
+ z[j++] = '?';
+ break;
+ }
+ case TK_LP:
+ case TK_RP: {
+ if( tokenType==TK_LP ){
+ nParen++;
+ if( prevTokenType==TK_IN ){
+ assert( nParen<nSql );
+ sqlite3HashInsert(&inHash, zSql+nParen, SQLITE_INT_TO_PTR(j));
+ }
+ }else{
+ int jj;
+ assert( nParen<nSql );
+ jj = SQLITE_PTR_TO_INT(sqlite3HashFind(&inHash, zSql+nParen));
+ if( jj>0 ){
+ sqlite3HashInsert(&inHash, zSql+nParen, 0);
+ assert( jj+6<nZ );
+ memcpy(z+jj+1, "?,?,?", 5);
+ j = jj+6;
+ assert( nZ-1-j>=0 );
+ assert( nZ-1-j<nZ );
+ memset(z+j, 0, nZ-1-j);
+ }
+ nParen--;
+ }
+ assert( nParen>=0 );
+ /* Fall through */
+ }
+ case TK_MINUS:
+ case TK_SEMI:
+ case TK_PLUS:
+ case TK_STAR:
+ case TK_SLASH:
+ case TK_REM:
+ case TK_EQ:
+ case TK_LE:
+ case TK_NE:
+ case TK_LSHIFT:
+ case TK_LT:
+ case TK_RSHIFT:
+ case TK_GT:
+ case TK_GE:
+ case TK_BITOR:
+ case TK_CONCAT:
+ case TK_COMMA:
+ case TK_BITAND:
+ case TK_BITNOT:
+ case TK_DOT:
+ case TK_IN:
+ case TK_IS:
+ case TK_NOT:
+ case TK_NULL:
+ case TK_ID: {
+ if( tokenType==TK_NULL ){
+ if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){
+ /* NULL is a keyword in this case, not a literal value */
+ }else{
+ /* Here the NULL is a literal value */
+ z[j++] = '?';
+ break;
+ }
+ }
+ if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
+ z[j++] = ' ';
+ }
+ if( tokenType==TK_ID ){
+ int i2 = i, n2 = n, rc = SQLITE_OK;
+ if( nParen>0 ){
+ assert( nParen<nSql );
+ sqlite3HashInsert(&inHash, zSql+nParen, 0);
+ }
+ if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
+ if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
+ if( rc!=SQLITE_OK ){
+ sqlite3DbFree(db, z);
+ sqlite3HashClear(&inHash);
+ return;
+ }
+ if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
+ z[j++] = '?';
+ break;
+ }
+ }
+ }
+ copyNormalizedToken(zSql, i, n, flags, z, &j);
+ break;
+ }
+ }
+ }
+ assert( j<nZ && "one" );
+ while( j>0 && z[j-1]==' ' ){ j--; }
+ if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
+ z[j] = 0;
+ assert( j<nZ && "two" );
+ pVdbe->zNormSql = z;
+ sqlite3HashClear(&inHash);
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
+
/*
** Rerun the compilation of a statement after a schema change.
**
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 2d090e3ec..9ead2b7aa 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -3609,9 +3609,19 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
** on this hint by avoiding the use of [lookaside memory] so as not to
** deplete the limited store of lookaside memory. Future versions of
** SQLite may act on this hint differently.
+**
+** [[SQLITE_PREPARE_NORMALIZE]] ^(<dt>SQLITE_PREPARE_NORMALIZE</dt>
+** <dd>The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized
+** representation of the SQL statement should be calculated and then
+** associated with the prepared statement, which can be obtained via
+** the [sqlite3_normalized_sql()] interface. The semantics used to
+** normalize a SQL statement are unspecified and subject to change.
+** At a minimum, literal values will be replaced with suitable
+** placeholders.
** </dl>
*/
#define SQLITE_PREPARE_PERSISTENT 0x01
+#define SQLITE_PREPARE_NORMALIZE 0x02
/*
** CAPI3REF: Compiling An SQL Statement
@@ -3769,6 +3779,11 @@ int sqlite3_prepare16_v3(
** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8
** string containing the SQL text of prepared statement P with
** [bound parameters] expanded.
+** ^The sqlite3_normalized_sql(P) interface returns a pointer to a UTF-8
+** string containing the normalized SQL text of prepared statement P. The
+** semantics used to normalize a SQL statement are unspecified and subject
+** to change. At a minimum, literal values will be replaced with suitable
+** placeholders.
**
** ^(For example, if a prepared statement is created using the SQL
** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345
@@ -3784,14 +3799,16 @@ int sqlite3_prepare16_v3(
** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time
** option causes sqlite3_expanded_sql() to always return NULL.
**
-** ^The string returned by sqlite3_sql(P) is managed by SQLite and is
-** automatically freed when the prepared statement is finalized.
+** ^The strings returned by sqlite3_sql(P) and sqlite3_normalized_sql(P)
+** are managed by SQLite and are automatically freed when the prepared
+** statement is finalized.
** ^The string returned by sqlite3_expanded_sql(P), on the other hand,
** is obtained from [sqlite3_malloc()] and must be free by the application
** by passing it to [sqlite3_free()].
*/
const char *sqlite3_sql(sqlite3_stmt *pStmt);
char *sqlite3_expanded_sql(sqlite3_stmt *pStmt);
+const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt);
/*
** CAPI3REF: Determine If An SQL Statement Writes The Database
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index 35d9950cf..34c41fd5a 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -310,12 +310,15 @@ struct sqlite3_api_routines {
int (*str_errcode)(sqlite3_str*);
int (*str_length)(sqlite3_str*);
char *(*str_value)(sqlite3_str*);
+ /* Version 3.25.0 and later */
int (*create_window_function)(sqlite3*,const char*,int,int,void*,
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*),
void (*xValue)(sqlite3_context*),
void (*xInv)(sqlite3_context*,int,sqlite3_value**),
void(*xDestroy)(void*));
+ /* Version 3.26.0 and later */
+ const char *(*normalized_sql)(sqlite3_stmt*);
};
/*
@@ -603,6 +606,8 @@ typedef int (*sqlite3_loadext_entry)(
#define sqlite3_str_value sqlite3_api->str_value
/* Version 3.25.0 and later */
#define sqlite3_create_window_function sqlite3_api->create_window_function
+/* Version 3.26.0 and later */
+#define sqlite3_normalized_sql sqlite3_api->normalized_sql
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index f0ed023c6..5a6042b2b 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1312,6 +1312,8 @@ struct FuncDefHash {
FuncDef *a[SQLITE_FUNC_HASH_SZ]; /* Hash table for functions */
};
+#define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ)
+
#ifdef SQLITE_USER_AUTHENTICATION
/*
** Information held in the "sqlite3" database connection object and used
@@ -1943,6 +1945,9 @@ struct VTable {
struct Table {
char *zName; /* Name of the table or view */
Column *aCol; /* Information about each column */
+#ifdef SQLITE_ENABLE_NORMALIZE
+ Hash *pColHash; /* All columns indexed by name */
+#endif
Index *pIndex; /* List of SQL indexes on this table. */
Select *pSelect; /* NULL for tables. Points to definition if a view. */
FKey *pFKey; /* Linked list of all foreign keys in this table */
@@ -2280,6 +2285,12 @@ struct IndexSample {
};
/*
+** Possible values to use within the flags argument to sqlite3GetToken().
+*/
+#define SQLITE_TOKEN_QUOTED 0x1 /* Token is a quoted identifier. */
+#define SQLITE_TOKEN_KEYWORD 0x2 /* Token is a keyword. */
+
+/*
** Each token coming out of the lexer is an instance of
** this structure. Tokens are also used as part of an expression.
**
@@ -4000,6 +4011,9 @@ int sqlite3ExprIsInteger(Expr*, int*);
int sqlite3ExprCanBeNull(const Expr*);
int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
int sqlite3IsRowid(const char*);
+#ifdef SQLITE_ENABLE_NORMALIZE
+int sqlite3IsRowidN(const char*, int);
+#endif
void sqlite3GenerateRowDelete(
Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
@@ -4026,6 +4040,9 @@ ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
IdList *sqlite3IdListDup(sqlite3*,IdList*);
Select *sqlite3SelectDup(sqlite3*,Select*,int);
+#ifdef SQLITE_ENABLE_NORMALIZE
+FuncDef *sqlite3FunctionSearchN(int,const char*,int);
+#endif
void sqlite3InsertBuiltinFuncs(FuncDef*,int);
FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
void sqlite3RegisterBuiltinFunctions(void);
@@ -4229,6 +4246,9 @@ void sqlite3AlterFunctions(void);
void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
int sqlite3GetToken(const unsigned char *, int *);
+#ifdef SQLITE_ENABLE_NORMALIZE
+int sqlite3GetTokenNormalized(const unsigned char *, int *, int *);
+#endif
void sqlite3NestedParse(Parse*, const char*, ...);
void sqlite3ExpirePreparedStatements(sqlite3*, int);
int sqlite3CodeSubselect(Parse*, Expr *, int, int);
@@ -4386,6 +4406,9 @@ sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
void sqlite3ParserReset(Parse*);
+#ifdef SQLITE_ENABLE_NORMALIZE
+void sqlite3Normalize(Vdbe*, const char*, int, u8);
+#endif
int sqlite3Reprepare(Vdbe*);
void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
diff --git a/src/test1.c b/src/test1.c
index d2c997bdf..7fee96924 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -4218,6 +4218,7 @@ static int SQLITE_TCLAPI test_prepare_v2(
char *zCopy = 0; /* malloc() copy of zSql */
int bytes;
const char *zTail = 0;
+ const char **pzTail;
sqlite3_stmt *pStmt = 0;
char zBuf[50];
int rc;
@@ -4242,7 +4243,8 @@ static int SQLITE_TCLAPI test_prepare_v2(
zCopy = malloc(n);
memcpy(zCopy, zSql, n);
}
- rc = sqlite3_prepare_v2(db, zCopy, bytes, &pStmt, objc>=5 ? &zTail : 0);
+ pzTail = objc>=5 ? &zTail : 0;
+ rc = sqlite3_prepare_v2(db, zCopy, bytes, &pStmt, pzTail);
free(zCopy);
zTail = &zSql[(zTail - zCopy)];
@@ -4270,6 +4272,79 @@ static int SQLITE_TCLAPI test_prepare_v2(
}
/*
+** Usage: sqlite3_prepare_v3 DB sql bytes flags ?tailvar?
+**
+** Compile up to <bytes> bytes of the supplied SQL string <sql> using
+** database handle <DB> and flags <flags>. The parameter <tailval> is
+** the name of a global variable that is set to the unused portion of
+** <sql> (if any). A STMT handle is returned.
+*/
+static int SQLITE_TCLAPI test_prepare_v3(
+ void * clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+ sqlite3 *db;
+ const char *zSql;
+ char *zCopy = 0; /* malloc() copy of zSql */
+ int bytes, flags;
+ const char *zTail = 0;
+ const char **pzTail;
+ sqlite3_stmt *pStmt = 0;
+ char zBuf[50];
+ int rc;
+
+ if( objc!=6 && objc!=5 ){
+ Tcl_AppendResult(interp, "wrong # args: should be \"",
+ Tcl_GetString(objv[0]), " DB sql bytes flags tailvar", 0);
+ return TCL_ERROR;
+ }
+ if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+ zSql = Tcl_GetString(objv[2]);
+ if( Tcl_GetIntFromObj(interp, objv[3], &bytes) ) return TCL_ERROR;
+ if( Tcl_GetIntFromObj(interp, objv[4], &flags) ) return TCL_ERROR;
+
+ /* Instead of using zSql directly, make a copy into a buffer obtained
+ ** directly from malloc(). The idea is to make it easier for valgrind
+ ** to spot buffer overreads. */
+ if( bytes>=0 ){
+ zCopy = malloc(bytes);
+ memcpy(zCopy, zSql, bytes);
+ }else{
+ int n = (int)strlen(zSql) + 1;
+ zCopy = malloc(n);
+ memcpy(zCopy, zSql, n);
+ }
+ pzTail = objc>=6 ? &zTail : 0;
+ rc = sqlite3_prepare_v3(db, zCopy, bytes, (unsigned int)flags,&pStmt,pzTail);
+ free(zCopy);
+ zTail = &zSql[(zTail - zCopy)];
+
+ assert(rc==SQLITE_OK || pStmt==0);
+ Tcl_ResetResult(interp);
+ if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR;
+ if( rc==SQLITE_OK && zTail && objc>=6 ){
+ if( bytes>=0 ){
+ bytes = bytes - (int)(zTail-zSql);
+ }
+ Tcl_ObjSetVar2(interp, objv[5], 0, Tcl_NewStringObj(zTail, bytes), 0);
+ }
+ if( rc!=SQLITE_OK ){
+ assert( pStmt==0 );
+ sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc);
+ Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0);
+ return TCL_ERROR;
+ }
+
+ if( pStmt ){
+ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
+ Tcl_AppendResult(interp, zBuf, 0);
+ }
+ return TCL_OK;
+}
+
+/*
** Usage: sqlite3_prepare_tkt3134 DB
**
** Generate a prepared statement for a zero-byte string as a test
@@ -4676,6 +4751,25 @@ static int SQLITE_TCLAPI test_ex_sql(
sqlite3_free(z);
return TCL_OK;
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+static int SQLITE_TCLAPI test_norm_sql(
+ void * clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+ sqlite3_stmt *pStmt;
+
+ if( objc!=2 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "STMT");
+ return TCL_ERROR;
+ }
+
+ if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
+ Tcl_SetResult(interp, (char *)sqlite3_normalized_sql(pStmt), TCL_VOLATILE);
+ return TCL_OK;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
/*
** Usage: sqlite3_column_count STMT
@@ -7646,6 +7740,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
{ "sqlite3_prepare", test_prepare ,0 },
{ "sqlite3_prepare16", test_prepare16 ,0 },
{ "sqlite3_prepare_v2", test_prepare_v2 ,0 },
+ { "sqlite3_prepare_v3", test_prepare_v3 ,0 },
{ "sqlite3_prepare_tkt3134", test_prepare_tkt3134, 0},
{ "sqlite3_prepare16_v2", test_prepare16_v2 ,0 },
{ "sqlite3_finalize", test_finalize ,0 },
@@ -7657,6 +7752,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
{ "sqlite3_step", test_step ,0 },
{ "sqlite3_sql", test_sql ,0 },
{ "sqlite3_expanded_sql", test_ex_sql ,0 },
+#ifdef SQLITE_ENABLE_NORMALIZE
+ { "sqlite3_normalized_sql", test_norm_sql ,0 },
+#endif
{ "sqlite3_next_stmt", test_next_stmt ,0 },
{ "sqlite3_stmt_readonly", test_stmt_readonly ,0 },
{ "sqlite3_stmt_busy", test_stmt_busy ,0 },
diff --git a/src/test_config.c b/src/test_config.c
index f017abc30..05002349b 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -762,6 +762,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
Tcl_SetVar2(interp, "sqlite_options", "uri_00_error", "0", TCL_GLOBAL_ONLY);
#endif
+#if defined(SQLITE_ENABLE_NORMALIZE)
+ Tcl_SetVar2(interp, "sqlite_options", "normalize", "1", TCL_GLOBAL_ONLY);
+#else
+ Tcl_SetVar2(interp, "sqlite_options", "normalize", "0", TCL_GLOBAL_ONLY);
+#endif
+
#ifdef SQLITE_OMIT_WINDOWFUNC
Tcl_SetVar2(interp, "sqlite_options", "windowfunc", "0", TCL_GLOBAL_ONLY);
#else
diff --git a/src/tokenize.c b/src/tokenize.c
index 262144ff7..05ca86e74 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -545,6 +545,73 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
return i;
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Return the length (in bytes) of the token that begins at z[0].
+** Store the token type in *tokenType before returning. If flags has
+** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type
+** for keywords. Add SQLITE_TOKEN_QUOTED to flags if the token was
+** actually a quoted identifier. Add SQLITE_TOKEN_KEYWORD to flags
+** if the token was recognized as a keyword; this is useful when the
+** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller
+** to differentiate between a keyword being treated as an identifier
+** (for normalization purposes) and an actual identifier.
+*/
+int sqlite3GetTokenNormalized(
+ const unsigned char *z,
+ int *tokenType,
+ int *flags
+){
+ int n;
+ unsigned char iClass = aiClass[*z];
+ if( iClass==CC_KYWD ){
+ int i;
+ for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+ if( IdChar(z[i]) ){
+ /* This token started out using characters that can appear in keywords,
+ ** but z[i] is a character not allowed within keywords, so this must
+ ** be an identifier instead */
+ i++;
+ while( IdChar(z[i]) ){ i++; }
+ *tokenType = TK_ID;
+ return i;
+ }
+ *tokenType = TK_ID;
+ n = keywordCode((char*)z, i, tokenType);
+ /* If the token is no longer considered to be an identifier, then it is a
+ ** keyword of some kind. Make the token back into an identifier and then
+ ** set the SQLITE_TOKEN_KEYWORD flag. Several non-identifier tokens are
+ ** used verbatim, including IN, IS, NOT, and NULL. */
+ switch( *tokenType ){
+ case TK_ID: {
+ /* do nothing, handled by caller */
+ break;
+ }
+ case TK_IN:
+ case TK_IS:
+ case TK_NOT:
+ case TK_NULL: {
+ *flags |= SQLITE_TOKEN_KEYWORD;
+ break;
+ }
+ default: {
+ *tokenType = TK_ID;
+ *flags |= SQLITE_TOKEN_KEYWORD;
+ break;
+ }
+ }
+ }else{
+ n = sqlite3GetToken(z, tokenType);
+ /* If the token is considered to be an identifier and the character class
+ ** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */
+ if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){
+ *flags |= SQLITE_TOKEN_QUOTED;
+ }
+ }
+ return n;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
+
/*
** Run the parser on the given SQL string. The parser structure is
** passed in. An SQLITE_ status code is returned. If an error occurs
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 107e5cab4..1cb721961 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -406,6 +406,9 @@ struct Vdbe {
yDbMask lockMask; /* Subset of btreeMask that requires a lock */
u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */
char *zSql; /* Text of the SQL statement that generated this */
+#ifdef SQLITE_ENABLE_NORMALIZE
+ char *zNormSql; /* Normalization of the associated SQL statement */
+#endif
void *pFree; /* Free this when deleting the vdbe */
VdbeFrame *pFrame; /* Parent frame */
VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index b21f70e7e..59327bed3 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -1702,6 +1702,16 @@ char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){
#endif
}
+#ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Return the normalized SQL associated with a prepared statement.
+*/
+const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
+ Vdbe *p = (Vdbe *)pStmt;
+ return p ? p->zNormSql : 0;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
+
#ifdef SQLITE_ENABLE_PREUPDATE_HOOK
/*
** Allocate and populate an UnpackedRecord structure based on the serialized
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 99df43596..f1496a3ab 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -64,6 +64,13 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){
}
assert( p->zSql==0 );
p->zSql = sqlite3DbStrNDup(p->db, z, n);
+#ifdef SQLITE_ENABLE_NORMALIZE
+ assert( p->zNormSql==0 );
+ if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
+ sqlite3Normalize(p, p->zSql, n, prepFlags);
+ assert( p->zNormSql!=0 || p->db->mallocFailed );
+ }
+#endif
}
/*
@@ -85,6 +92,11 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
zTmp = pA->zSql;
pA->zSql = pB->zSql;
pB->zSql = zTmp;
+#ifdef SQLITE_ENABLE_NORMALIZE
+ zTmp = pA->zNormSql;
+ pA->zNormSql = pB->zNormSql;
+ pB->zNormSql = zTmp;
+#endif
pB->expmask = pA->expmask;
pB->prepFlags = pA->prepFlags;
memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter));
@@ -3156,6 +3168,9 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
vdbeFreeOpArray(db, p->aOp, p->nOp);
sqlite3DbFree(db, p->aColName);
sqlite3DbFree(db, p->zSql);
+#ifdef SQLITE_ENABLE_NORMALIZE
+ sqlite3DbFree(db, p->zNormSql);
+#endif
#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
{
int i;
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-20 18:00:09 +0000