Optimizations in the tokenizer. (CVS 1985)

FossilOrigin-Name: 26898c57cb2419d4200803f79fdd821c3093cba2

3 files changed, 39 insertions, 42 deletions

diff --git a/manifest b/manifest
index 0b0820fad..f3ac487d9 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Code\scleanup:\sget\srid\sof\sthe\ssqlite3SetNString\sutility\sfunction.\s(CVS\s1984)
-D 2004-09-25T14:39:18
+C Optimizations\sin\sthe\stokenizer.\s(CVS\s1985)
+D 2004-09-25T15:25:26
 F Makefile.in abdeb5bd9d017822691884935c320037c33f6ee6
 F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457
 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -68,7 +68,7 @@ F src/test2.c 0f3e0ad7b675a6f3323211ab4ea95490855654c3
 F src/test3.c 5b5b0f3d11b097399c1054fff73d8f3711092301
 F src/test4.c 7c6b9fc33dd1f3f93c7f1ee6e5e6d016afa6c1df
 F src/test5.c b001fa7f1b9e2dc5c2331de62fc641b5ab2bd7a1
-F src/tokenize.c 85a5b0de4635bc96ec9860274449448ff0d0726d
+F src/tokenize.c d4619367d9ba17c6bd4e018fb7e91015ae8562aa
 F src/trigger.c d1f770ee37a80391dd6d0948ee821b0272f99ae7
 F src/update.c 7157084216c4b02a23cdb23eb6d246aa9034fa4d
 F src/utf.c f4f83acd73389090e32d6589d307fc55d794c7ed
@@ -247,7 +247,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25
 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9
 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
 F www/whentouse.tcl a8335bce47cc2fddb07f19052cb0cb4d9129a8e4
-P cb631a135da92fd689ee40a8b7e2695a510eb765
-R 0e57f8bf788b1fab54d791e8a0ff38cb
+P 9ef4c24a9acc2128891303de1ffd2ef4509d779c
+R 5ed5520ba8ef2bfa197e6a975f384e30
 U drh
-Z 4e1f4e3ae25eddb22e2a4ce1e3a8ec1c
+Z 610f3581cf05ef77bbf5335c4f0a0443
diff --git a/manifest.uuid b/manifest.uuid
index d7bf26012..df27ad4cc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9ef4c24a9acc2128891303de1ffd2ef4509d779c
\ No newline at end of file
+26898c57cb2419d4200803f79fdd821c3093cba2
\ No newline at end of file
diff --git a/src/tokenize.c b/src/tokenize.c
index 7e2ab6274..5424eb047 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -15,7 +15,7 @@
 ** individual tokens and sends those tokens one-by-one over to the
 ** parser for analysis.
 **
-** $Id: tokenize.c,v 1.88 2004/09/25 14:39:19 drh Exp $
+** $Id: tokenize.c,v 1.89 2004/09/25 15:25:26 drh Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -52,7 +52,6 @@ static Keyword aKeywordTable[] = {
   { "CASCADE",           TK_CASCADE,      },
   { "CASE",              TK_CASE,         },
   { "CHECK",             TK_CHECK,        },
-  { "CLUSTER",           TK_CLUSTER,      },
   { "COLLATE",           TK_COLLATE,      },
   { "COMMIT",            TK_COMMIT,       },
   { "CONFLICT",          TK_CONFLICT,     },
@@ -160,11 +159,11 @@ int sqlite3KeywordCode(const char *z, int n){
     if( needInit ){
       int nk;
       nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]);
-      for(i=0; i<nk; i++){
-        aKeywordTable[i].len = strlen(aKeywordTable[i].zName);
-        h = sqlite3HashNoCase(aKeywordTable[i].zName, aKeywordTable[i].len);
-        h %= KEY_HASH_SIZE;
-        aKeywordTable[i].iNext = aiHashTable[h];
+      for(i=0, p=aKeywordTable; i<nk; i++, p++){
+        const char *zName = p->zName;
+        int len = p->len = strlen(zName);
+        h = sqlite3HashNoCase(zName, len) % KEY_HASH_SIZE;
+        p->iNext = aiHashTable[h];
         aiHashTable[h] = i+1;
       }
       needInit = 0;
@@ -198,9 +197,6 @@ int sqlite3KeywordCode(const char *z, int n){
 */
 static const char isIdChar[] = {
 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
@@ -208,13 +204,14 @@ static const char isIdChar[] = {
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
 };
 
+#define IdChar(C)  (((c=C)&0x80)!=0 || (c>0x2f && isIdChar[c-0x30]))
 
 /*
 ** Return the length of the token that begins at z[0]. 
 ** Store the token type in *tokenType before returning.
 */
 static int sqliteGetToken(const unsigned char *z, int *tokenType){
-  int i;
+  int i, c;
   switch( *z ){
     case ' ': case '\t': case '\n': case '\f': case '\r': {
       for(i=1; isspace(z[i]); i++){}
@@ -223,7 +220,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
     }
     case '-': {
       if( z[1]=='-' ){
-        for(i=2; z[i] && z[i]!='\n'; i++){}
+        for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
         *tokenType = TK_COMMENT;
         return i;
       }
@@ -255,8 +252,8 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
         *tokenType = TK_SLASH;
         return 1;
       }
-      for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
-      if( z[i] ) i++;
+      for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
+      if( c ) i++;
       *tokenType = TK_COMMENT;
       return i;
     }
@@ -269,13 +266,13 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
       return 1 + (z[1]=='=');
     }
     case '<': {
-      if( z[1]=='=' ){
+      if( (c=z[1])=='=' ){
         *tokenType = TK_LE;
         return 2;
-      }else if( z[1]=='>' ){
+      }else if( c=='>' ){
         *tokenType = TK_NE;
         return 2;
-      }else if( z[1]=='<' ){
+      }else if( c=='<' ){
         *tokenType = TK_LSHIFT;
         return 2;
       }else{
@@ -284,10 +281,10 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
       }
     }
     case '>': {
-      if( z[1]=='=' ){
+      if( (c=z[1])=='=' ){
         *tokenType = TK_GE;
         return 2;
-      }else if( z[1]=='>' ){
+      }else if( c=='>' ){
         *tokenType = TK_RSHIFT;
         return 2;
       }else{
@@ -327,8 +324,8 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
     }
     case '\'': case '"': {
       int delim = z[0];
-      for(i=1; z[i]; i++){
-        if( z[i]==delim ){
+      for(i=1; (c=z[i])!=0; i++){
+        if( c==delim ){
           if( z[i+1]==delim ){
             i++;
           }else{
@@ -336,7 +333,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
           }
         }
       }
-      if( z[i] ) i++;
+      if( c ) i++;
       *tokenType = TK_STRING;
       return i;
     }
@@ -365,7 +362,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
       return i;
     }
     case '[': {
-      for(i=1; z[i] && z[i-1]!=']'; i++){}
+      for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
       *tokenType = TK_ID;
       return i;
     }
@@ -375,12 +372,11 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
       return i;
     }
     case ':': {
-      for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){}
+      for(i=1; IdChar(z[i]); i++){}
       *tokenType = i>1 ? TK_VARIABLE : TK_ILLEGAL;
       return i;
     }
     case '$': {
-      int c;
       *tokenType = TK_VARIABLE;
       if( z[1]=='{' ){
         int nBrace = 1;
@@ -418,29 +414,29 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
       return i;
     } 
     case 'x': case 'X': {
-      if( z[1]=='\'' || z[1]=='"' ){
-        int delim = z[1];
+      if( (c=z[1])=='\'' || c=='"' ){
+        int delim = c;
         *tokenType = TK_BLOB;
-        for(i=2; z[i]; i++){
-          if( z[i]==delim ){
+        for(i=2; (c=z[i])!=0; i++){
+          if( c==delim ){
             if( i%2 ) *tokenType = TK_ILLEGAL;
             break;
           }
-          if( !isxdigit(z[i]) ){
+          if( !isxdigit(c) ){
             *tokenType = TK_ILLEGAL;
             return i;
           }
         }
-        if( z[i] ) i++;
+        if( c ) i++;
         return i;
       }
       /* Otherwise fall through to the next case */
     }
     default: {
-      if( (*z&0x80)==0 && !isIdChar[*z] ){
+      if( !IdChar(*z) ){
         break;
       }
-      for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){}
+      for(i=1; IdChar(z[i]); i++){}
       *tokenType = sqlite3KeywordCode((char*)z, i);
       return i;
     }
@@ -692,10 +688,11 @@ int sqlite3_complete(const char *zSql){
         break;
       }
       default: {
-        if( isIdChar[(u8)*zSql] ){
+        int c;
+        if( IdChar((u8)*zSql) ){
           /* Keywords and unquoted identifiers */
           int nId;
-          for(nId=1; isIdChar[(u8)zSql[nId]]; nId++){}
+          for(nId=1; IdChar(zSql[nId]); nId++){}
           switch( *zSql ){
             case 'c': case 'C': {
               if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){