Fix an obscure race condition that can occur when multiple threads, shared cache and DDL statements are combined. Enhance notify2.test to test this scenario. (CVS 6373)

FossilOrigin-Name: 92ec5975123284aff3a69ee16c397d9e2a844c0b

6 files changed, 113 insertions, 59 deletions

diff --git a/manifest b/manifest
index e0393068c..6bbbfd8c5 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sthe\sROUND8()\smacro\sto\sround\san\sinteger\sup\sto\sthe\snearest\smultiple\sof\s8\sand\sROUNDDOWN8()\smacro\sto\sround\sdown\sto\sthe\snearest\smultiple\sof\s8.\sThis\sis\sa\scosmetic\schange.\s(CVS\s6372)
-D 2009-03-23T04:33:32
+C Fix\san\sobscure\srace\scondition\sthat\scan\soccur\swhen\smultiple\sthreads,\sshared\scache\sand\sDDL\sstatements\sare\scombined.\sEnhance\snotify2.test\sto\stest\sthis\sscenario.\s(CVS\s6373)
+D 2009-03-23T17:11:27
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 583e87706abc3026960ed759aff6371faf84c211
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -150,7 +150,7 @@ F src/pcache.c fcf7738c83c4d3e9d45836b2334c8a368cc41274
 F src/pcache.h 9b927ccc5a538e31b4c3bc7eec4f976db42a1324
 F src/pcache1.c f587565f4ba0fd1772067eaa96814dce761b7a4c
 F src/pragma.c 22ed04836aab8ce134c53be1ca896f3ad20fabdb
-F src/prepare.c 14e61702b09a325e50000e5f5f3156cd20c4afe1
+F src/prepare.c db64c97583b59d78a9eebd2bfc3e61d9f571ef12
 F src/printf.c 9866a9a9c4a90f6d4147407f373df3fd5d5f9b6f
 F src/random.c 676b9d7ac820fe81e6fb2394ac8c10cff7f38628
 F src/resolve.c 094e44450371fb27869eb8bf679aacbe51fdc56d
@@ -192,7 +192,7 @@ F src/test_pcache.c 29464896d9c67832e4eef916c0682b98d7283d00
 F src/test_schema.c 4b4bf7bb329326458c491b0e6facd4c8c4c5b479
 F src/test_server.c f0a403b5f699c09bd2b1236b6f69830fd6221f6b
 F src/test_tclvar.c 9e42fa59d3d2f064b7ab8628e7ab2dc8a9fe93d4
-F src/test_thread.c 870a862d9c740d083b93ed30a0b5c0b491b30f43
+F src/test_thread.c 1c7a895dbc469d87d22c1b36f269a6f799fc47a8
 F src/test_wsd.c 3ae5101de6cbfda2720152ab659ea84079719241
 F src/tokenize.c 6987fb7f0d6a87ac53499aee568cabb05eb4bea8
 F src/trigger.c 21f39db410cdc32166a94900ac1b3df98ea560e6
@@ -200,7 +200,7 @@ F src/update.c 8ededddcde6f7b6da981dd0429a5d34518a475b7
 F src/utf.c 1da9c832dba0fa8f865b5b902d93f420a1ee4245
 F src/util.c 469d74f5bf09ed6398702c7da2ef8a34e979a1c1
 F src/vacuum.c 4929a585ef0fb1dfaf46302f8a9c4aa30c2d9cf5
-F src/vdbe.c f8164c2830f82714a77b1f2a97c2e9c4efbcb3bb
+F src/vdbe.c 624922027b8b5fe203bd89e204aaed447e8b7ce7
 F src/vdbe.h d70a68bee196ab228914a3902c79dbd24342a0f2
 F src/vdbeInt.h 53a2f4696871712646c77351904576cca6ad9752
 F src/vdbeapi.c 025d83f51f5e0a6e0fb55d603103d24825577967
@@ -479,7 +479,7 @@ F test/mutex1.test ace3ba551ef0cd864df2b820b5dbe18d28b4db6c
 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test c627d79b3d36ea892563fd67584b3e8a18f0618a
 F test/notify1.test 9a985a94f34de1b24daf25fd86b6d5033ba532d0
-F test/notify2.test 0c350ad8ff7d7d6ceb24ac84f33df0ce481f86e1
+F test/notify2.test 997b6caed0c4e2f9549a30eb5df240d0df298eba
 F test/notnull.test 44d600f916b770def8b095a9962dbe3be5a70d82
 F test/null.test a8b09b8ed87852742343b33441a9240022108993
 F test/openv2.test f5dd6b23e4dce828eb211649b600763c42a668df
@@ -709,7 +709,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P 83a4d5b3d7abe983c2be00a694cc7d4c4847f253
-R e8c1b980a9474b2681755e6236a2eed3
+P db1d4d2f5083adf5438c7f387b115180800e7bd9
+R 7eb80d4e0dc8a8ee68d4260f71752688
 U danielk1977
-Z af91f276a825722179e40ac35b5fa895
+Z f6209142c4d9c04d943d837539d0d12b
diff --git a/manifest.uuid b/manifest.uuid
index a50af64cf..ffa1407f3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-db1d4d2f5083adf5438c7f387b115180800e7bd9
\ No newline at end of file
+92ec5975123284aff3a69ee16c397d9e2a844c0b
\ No newline at end of file
diff --git a/src/prepare.c b/src/prepare.c
index 3367f7979..282e46a84 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -13,7 +13,7 @@
 ** interface, and routines that contribute to loading the database schema
 ** from disk.
 **
-** $Id: prepare.c,v 1.111 2009/03/19 18:51:07 danielk1977 Exp $
+** $Id: prepare.c,v 1.112 2009/03/23 17:11:27 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 
@@ -91,7 +91,7 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){
       pData->rc = rc;
       if( rc==SQLITE_NOMEM ){
         db->mallocFailed = 1;
-      }else if( rc!=SQLITE_INTERRUPT ){
+      }else if( rc!=SQLITE_INTERRUPT && (rc&0xff)!=SQLITE_LOCKED ){
         corruptSchema(pData, argv[0], zErr);
       }
       sqlite3DbFree(db, zErr);
diff --git a/src/test_thread.c b/src/test_thread.c
index 20fcf73a2..ce51a6e05 100644
--- a/src/test_thread.c
+++ b/src/test_thread.c
@@ -14,7 +14,7 @@
 ** test that sqlite3 database handles may be concurrently accessed by 
 ** multiple threads. Right now this only works on unix.
 **
-** $Id: test_thread.c,v 1.12 2009/03/19 07:58:31 danielk1977 Exp $
+** $Id: test_thread.c,v 1.13 2009/03/23 17:11:27 danielk1977 Exp $
 */
 
 #include "sqliteInt.h"
@@ -117,8 +117,10 @@ static Tcl_ThreadCreateType tclScriptThread(ClientData pSqlThread){
   Tcl_CreateObjCommand(interp, "sqlthread", sqlthread_proc, pSqlThread, 0);
 #if defined(OS_UNIX) && defined(SQLITE_ENABLE_UNLOCK_NOTIFY)
   Tcl_CreateObjCommand(interp, "sqlite3_blocking_step", blocking_step_proc,0,0);
-  Tcl_CreateObjCommand(
-      interp, "sqlite3_blocking_prepare_v2", blocking_prepare_v2_proc,0,0);
+  Tcl_CreateObjCommand(interp, 
+      "sqlite3_blocking_prepare_v2", blocking_prepare_v2_proc, (void *)1, 0);
+  Tcl_CreateObjCommand(interp, 
+      "sqlite3_nonblocking_prepare_v2", blocking_prepare_v2_proc, 0, 0);
 #endif
   Sqlitetest1_Init(interp);
   Sqlitetest_mutex_Init(interp);
@@ -544,6 +546,7 @@ static int blocking_step_proc(
 
 /*
 ** Usage: sqlite3_blocking_prepare_v2 DB sql bytes ?tailvar?
+** Usage: sqlite3_nonblocking_prepare_v2 DB sql bytes ?tailvar?
 */
 static int blocking_prepare_v2_proc(
   void * clientData,
@@ -558,6 +561,7 @@ static int blocking_prepare_v2_proc(
   sqlite3_stmt *pStmt = 0;
   char zBuf[50];
   int rc;
+  int isBlocking = !(clientData==0);
 
   if( objc!=5 && objc!=4 ){
     Tcl_AppendResult(interp, "wrong # args: should be \"", 
@@ -568,7 +572,12 @@ static int blocking_prepare_v2_proc(
   zSql = Tcl_GetString(objv[2]);
   if( Tcl_GetIntFromObj(interp, objv[3], &bytes) ) return TCL_ERROR;
 
-  rc = sqlite3_blocking_prepare_v2(db, zSql, bytes, &pStmt, objc>=5?&zTail : 0);
+  if( isBlocking ){
+    rc = sqlite3_blocking_prepare_v2(db, zSql, bytes, &pStmt, &zTail);
+  }else{
+    rc = sqlite3_prepare_v2(db, zSql, bytes, &pStmt, &zTail);
+  }
+
   assert(rc==SQLITE_OK || pStmt==0);
   if( zTail && objc>=5 ){
     if( bytes>=0 ){
@@ -603,8 +612,10 @@ int SqlitetestThread_Init(Tcl_Interp *interp){
   Tcl_CreateObjCommand(interp, "clock_seconds", clock_seconds_proc, 0, 0);
 #if defined(OS_UNIX) && defined(SQLITE_ENABLE_UNLOCK_NOTIFY)
   Tcl_CreateObjCommand(interp, "sqlite3_blocking_step", blocking_step_proc,0,0);
-  Tcl_CreateObjCommand(
-      interp, "sqlite3_blocking_prepare_v2", blocking_prepare_v2_proc,0,0);
+  Tcl_CreateObjCommand(interp, 
+      "sqlite3_blocking_prepare_v2", blocking_prepare_v2_proc, (void *)1, 0);
+  Tcl_CreateObjCommand(interp, 
+      "sqlite3_nonblocking_prepare_v2", blocking_prepare_v2_proc, 0, 0);
 #endif
   return TCL_OK;
 }
diff --git a/src/vdbe.c b/src/vdbe.c
index 97f5ae696..128e81bdf 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -43,7 +43,7 @@
 ** in this file for details.  If in doubt, do not deviate from existing
 ** commenting and indentation practices when changing or adding code.
 **
-** $Id: vdbe.c,v 1.827 2009/03/18 10:33:02 danielk1977 Exp $
+** $Id: vdbe.c,v 1.828 2009/03/23 17:11:27 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
@@ -4371,33 +4371,58 @@ case OP_CreateTable: {          /* out2-prerelease */
 ** then runs the new virtual machine.  It is thus a re-entrant opcode.
 */
 case OP_ParseSchema: {
-  char *zSql;
   int iDb = pOp->p1;
-  const char *zMaster;
-  InitData initData;
-
   assert( iDb>=0 && iDb<db->nDb );
-  if( !pOp->p2 && !DbHasProperty(db, iDb, DB_SchemaLoaded) ){
-    break;
+
+  /* If pOp->p2 is 0, then this opcode is being executed to read a
+  ** single row, for example the row corresponding to a new index
+  ** created by this VDBE, from the sqlite_master table. It only
+  ** does this if the corresponding in-memory schema is currently
+  ** loaded. Otherwise, the new index definition can be loaded along
+  ** with the rest of the schema when it is required.
+  **
+  ** Although the mutex on the BtShared object that corresponds to
+  ** database iDb (the database containing the sqlite_master table
+  ** read by this instruction) is currently held, it is necessary to
+  ** obtain the mutexes on all attached databases before checking if
+  ** the schema of iDb is loaded. This is because, at the start of
+  ** the sqlite3_exec() call below, SQLite will invoke 
+  ** sqlite3BtreeEnterAll(). If all mutexes are not already held, the
+  ** iDb mutex may be temporarily released to avoid deadlock. If 
+  ** this happens, then some other thread may delete the in-memory 
+  ** schema of database iDb before the SQL statement runs. The schema
+  ** will not be reloaded becuase the db->init.busy flag is set. This
+  ** can result in a "no such table: sqlite_master" or "malformed
+  ** database schema" error being returned to the user.
+  */
+  assert( sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) );
+  sqlite3BtreeEnterAll(db);
+  if( pOp->p2 || DbHasProperty(db, iDb, DB_SchemaLoaded) ){
+    const char *zMaster = SCHEMA_TABLE(iDb);
+    char *zSql;
+    InitData initData;
+    initData.db = db;
+    initData.iDb = pOp->p1;
+    initData.pzErrMsg = &p->zErrMsg;
+    zSql = sqlite3MPrintf(db,
+       "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s",
+       db->aDb[iDb].zName, zMaster, pOp->p4.z);
+    if( zSql==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      (void)sqlite3SafetyOff(db);
+      assert( db->init.busy==0 );
+      db->init.busy = 1;
+      initData.rc = SQLITE_OK;
+      assert( !db->mallocFailed );
+      rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0);
+      if( rc==SQLITE_OK ) rc = initData.rc;
+      sqlite3DbFree(db, zSql);
+      db->init.busy = 0;
+      (void)sqlite3SafetyOn(db);
+    }
   }
-  zMaster = SCHEMA_TABLE(iDb);
-  initData.db = db;
-  initData.iDb = pOp->p1;
-  initData.pzErrMsg = &p->zErrMsg;
-  zSql = sqlite3MPrintf(db,
-     "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s",
-     db->aDb[iDb].zName, zMaster, pOp->p4.z);
-  if( zSql==0 ) goto no_mem;
-  (void)sqlite3SafetyOff(db);
-  assert( db->init.busy==0 );
-  db->init.busy = 1;
-  initData.rc = SQLITE_OK;
-  assert( !db->mallocFailed );
-  rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0);
-  if( rc==SQLITE_OK ) rc = initData.rc;
-  sqlite3DbFree(db, zSql);
-  db->init.busy = 0;
-  (void)sqlite3SafetyOn(db);
+  sqlite3BtreeLeaveAll(db);
   if( rc==SQLITE_NOMEM ){
     goto no_mem;
   }
diff --git a/test/notify2.test b/test/notify2.test
index 8b508b2bc..a079c36f2 100644
--- a/test/notify2.test
+++ b/test/notify2.test
@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 #
-# $Id: notify2.test,v 1.2 2009/03/19 07:58:31 danielk1977 Exp $
+# $Id: notify2.test,v 1.3 2009/03/23 17:11:27 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -62,6 +62,8 @@ set ThreadProgram {
     set lRes [list]
     set rc SQLITE_OK
 
+set sql $zSql
+
     while {$rc=="SQLITE_OK" && $zSql ne ""} {
       set STMT [$::xPrepare $db $zSql -1 zSql]
       while {[set rc [$::xStep $STMT]] eq "SQLITE_ROW"} {
@@ -72,22 +74,33 @@ set ThreadProgram {
       set rc [sqlite3_finalize $STMT]
     }
 
-    if {$rc != "SQLITE_OK"} { error "$rc [sqlite3_errmsg $db]" }
+    if {$rc != "SQLITE_OK"} { error "$rc $sql [sqlite3_errmsg $db]" }
     return $lRes
   }
 
+  proc execsql_retry {db sql} { 
+    set msg "SQLITE_LOCKED blah..."
+    while { [string match SQLITE_LOCKED* $msg] } {
+      catch { execsql_blocking $db $sql } msg
+    }
+  }
+
   proc select_one {args} {
     set n [llength $args]
     lindex $args [expr int($n*rand())]
   }
 
-  # Open a database connection. Attach the two auxillary databases.
-  set ::DB [sqlite3_open test.db]
-  execsql_blocking $::DB {
-    ATTACH 'test2.db' AS aux2;
-    ATTACH 'test3.db' AS aux3;
+  proc opendb {} {
+    # Open a database connection. Attach the two auxillary databases.
+    set ::DB [sqlite3_open test.db]
+    execsql_retry $::DB { ATTACH 'test2.db' AS aux2; }
+    execsql_retry $::DB { ATTACH 'test3.db' AS aux3; }
   }
 
+  opendb
+
+  #after 2000
+
   # This loop runs for ~20 seconds.
   #
   set iStart [clock_seconds]
@@ -108,12 +121,11 @@ set ThreadProgram {
       } {
             DELETE FROM xxx WHERE a<(SELECT max(a)-100 FROM xxx);
             INSERT INTO xxx SELECT NULL, total(a) FROM xxx;
-      } 
-#      {
-#            CREATE INDEX IF NOT EXISTS yyy.xxx_i ON xxx(b);
-#      } {
-#            DROP INDEX IF EXISTS yyy.xxx_i;
-#      }
+      } {
+            CREATE INDEX IF NOT EXISTS yyy.xxx_i ON xxx(b);
+      } {
+            DROP INDEX IF EXISTS yyy.xxx_i;
+      }
       ]]
     }
 
@@ -128,9 +140,15 @@ set ThreadProgram {
       "
     } msg]
 
-    if {$rc && [string match "SQLITE_LOCKED*" $msg]} {
+    if {$rc && [string match "SQLITE_LOCKED*" $msg]
+            || [string match "SQLITE_SCHEMA*" $msg]
+    } {
       # Hit an SQLITE_LOCKED error. Rollback the current transaction.
-      execsql_blocking $::DB ROLLBACK
+      set rc [catch { execsql_blocking $::DB ROLLBACK } msg]
+      if {$rc && [string match "SQLITE_LOCKED*" $msg]} {
+        sqlite3_close $::DB
+        opendb
+      } 
     } elseif {$rc} {
       # Hit some other kind of error. This is a malfunction.
       error $msg
@@ -150,7 +168,7 @@ set ThreadProgram {
 
 foreach {iTest xStep xPrepare} {
   1 sqlite3_blocking_step sqlite3_blocking_prepare_v2
-  2 sqlite3_step          sqlite3_prepare_v2
+  2 sqlite3_step          sqlite3_nonblocking_prepare_v2
 } {
   file delete -force test.db test2.db test3.db