Changes so that test_async.c works with memory management turned on. (CVS 3093)

FossilOrigin-Name: f4150c29df2774b4422d4296d913cdbcee62c859

6 files changed, 103 insertions, 47 deletions

diff --git a/src/hash.c b/src/hash.c
index 2ef86a84f..f15613e14 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -12,7 +12,7 @@
 ** This is the implementation of generic hash-tables
 ** used in SQLite.
 **
-** $Id: hash.c,v 1.17 2005/10/03 15:11:09 drh Exp $
+** $Id: hash.c,v 1.18 2006/02/14 10:48:39 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include <assert.h>
@@ -41,6 +41,8 @@ void sqlite3HashInit(Hash *pNew, int keyClass, int copyKey){
   pNew->count = 0;
   pNew->htsize = 0;
   pNew->ht = 0;
+  pNew->xMalloc = sqlite3MallocX;
+  pNew->xFree = sqlite3FreeX;
 }
 
 /* Remove all entries from a hash table.  Reclaim all memory.
@@ -53,15 +55,15 @@ void sqlite3HashClear(Hash *pH){
   assert( pH!=0 );
   elem = pH->first;
   pH->first = 0;
-  if( pH->ht ) sqliteFree(pH->ht);
+  if( pH->ht ) pH->xFree(pH->ht);
   pH->ht = 0;
   pH->htsize = 0;
   while( elem ){
     HashElem *next_elem = elem->next;
     if( pH->copyKey && elem->pKey ){
-      sqliteFree(elem->pKey);
+      pH->xFree(elem->pKey);
     }
-    sqliteFree(elem);
+    pH->xFree(elem);
     elem = next_elem;
   }
   pH->count = 0;
@@ -222,9 +224,9 @@ static void rehash(Hash *pH, int new_size){
   int (*xHash)(const void*,int); /* The hash function */
 
   assert( (new_size & (new_size-1))==0 );
-  new_ht = (struct _ht *)sqliteMalloc( new_size*sizeof(struct _ht) );
+  new_ht = (struct _ht *)pH->xMalloc( new_size*sizeof(struct _ht) );
   if( new_ht==0 ) return;
-  if( pH->ht ) sqliteFree(pH->ht);
+  if( pH->ht ) pH->xFree(pH->ht);
   pH->ht = new_ht;
   pH->htsize = new_size;
   xHash = hashFunction(pH->keyClass);
@@ -290,9 +292,9 @@ static void removeElementGivenHash(
     pEntry->chain = 0;
   }
   if( pH->copyKey && elem->pKey ){
-    sqliteFree(elem->pKey);
+    pH->xFree(elem->pKey);
   }
-  sqliteFree( elem );
+  pH->xFree( elem );
   pH->count--;
   if( pH->count<=0 ){
     assert( pH->first==0 );
@@ -358,12 +360,12 @@ void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
     return old_data;
   }
   if( data==0 ) return 0;
-  new_elem = (HashElem*)sqliteMalloc( sizeof(HashElem) );
+  new_elem = (HashElem*)pH->xMalloc( sizeof(HashElem) );
   if( new_elem==0 ) return data;
   if( pH->copyKey && pKey!=0 ){
-    new_elem->pKey = sqliteMallocRaw( nKey );
+    new_elem->pKey = pH->xMalloc( nKey );
     if( new_elem->pKey==0 ){
-      sqliteFree(new_elem);
+      pH->xFree(new_elem);
       return data;
     }
     memcpy((void*)new_elem->pKey, pKey, nKey);
@@ -376,7 +378,7 @@ void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
     rehash(pH,8);
     if( pH->htsize==0 ){
       pH->count = 0;
-      sqliteFree(new_elem);
+      pH->xFree(new_elem);
       return data;
     }
   }
diff --git a/src/hash.h b/src/hash.h
index aa17828a6..78f60b4b9 100644
--- a/src/hash.h
+++ b/src/hash.h
@@ -12,7 +12,7 @@
 ** This is the header file for the generic hash-table implemenation
 ** used in SQLite.
 **
-** $Id: hash.h,v 1.8 2004/08/20 14:08:51 drh Exp $
+** $Id: hash.h,v 1.9 2006/02/14 10:48:39 danielk1977 Exp $
 */
 #ifndef _SQLITE_HASH_H_
 #define _SQLITE_HASH_H_
@@ -34,6 +34,8 @@ struct Hash {
   char copyKey;           /* True if copy of key made on insert */
   int count;              /* Number of entries in this table */
   HashElem *first;        /* The first element of the array */
+  void *(*xMalloc)(int);  /* malloc() function to use */
+  void (*xFree)(void *);  /* free() function to use */
   int htsize;             /* Number of buckets in the hash table */
   struct _ht {            /* the hash table */
     int count;               /* Number of entries with this hash */
diff --git a/src/os_unix.c b/src/os_unix.c
index ce3101ed1..c61efb81e 100644
--- a/src/os_unix.c
+++ b/src/os_unix.c
@@ -340,9 +340,10 @@ struct openCnt {
 ** openKey structures) into lockInfo and openCnt structures.  Access to 
 ** these hash tables must be protected by a mutex.
 */
-static Hash lockHash = { SQLITE_HASH_BINARY, 0, 0, 0, 0, 0 };
-static Hash openHash = { SQLITE_HASH_BINARY, 0, 0, 0, 0, 0 };
-
+static Hash lockHash = {SQLITE_HASH_BINARY, 0, 0, 0, 
+    sqlite3ThreadSafeMalloc, sqlite3ThreadSafeFree, 0, 0};
+static Hash openHash = {SQLITE_HASH_BINARY, 0, 0, 0, 
+    sqlite3ThreadSafeMalloc, sqlite3ThreadSafeFree, 0, 0};
 
 #ifdef SQLITE_UNIX_THREADS
 /*
@@ -491,7 +492,7 @@ static void releaseLockInfo(struct lockInfo *pLock){
   pLock->nRef--;
   if( pLock->nRef==0 ){
     sqlite3HashInsert(&lockHash, &pLock->key, sizeof(pLock->key), 0);
-    sqliteFree(pLock);
+    sqlite3ThreadSafeFree(pLock);
   }
 }
 
@@ -504,7 +505,7 @@ static void releaseOpenCnt(struct openCnt *pOpen){
   if( pOpen->nRef==0 ){
     sqlite3HashInsert(&openHash, &pOpen->key, sizeof(pOpen->key), 0);
     free(pOpen->aPending);
-    sqliteFree(pOpen);
+    sqlite3ThreadSafeFree(pOpen);
   }
 }
 
@@ -545,7 +546,7 @@ static int findLockInfo(
   pLock = (struct lockInfo*)sqlite3HashFind(&lockHash, &key1, sizeof(key1));
   if( pLock==0 ){
     struct lockInfo *pOld;
-    pLock = sqliteMallocRaw( sizeof(*pLock) );
+    pLock = sqlite3ThreadSafeMalloc( sizeof(*pLock) );
     if( pLock==0 ){
       rc = 1;
       goto exit_findlockinfo;
@@ -557,7 +558,7 @@ static int findLockInfo(
     pOld = sqlite3HashInsert(&lockHash, &pLock->key, sizeof(key1), pLock);
     if( pOld!=0 ){
       assert( pOld==pLock );
-      sqliteFree(pLock);
+      sqlite3ThreadSafeFree(pLock);
       rc = 1;
       goto exit_findlockinfo;
     }
@@ -569,7 +570,7 @@ static int findLockInfo(
     pOpen = (struct openCnt*)sqlite3HashFind(&openHash, &key2, sizeof(key2));
     if( pOpen==0 ){
       struct openCnt *pOld;
-      pOpen = sqliteMallocRaw( sizeof(*pOpen) );
+      pOpen = sqlite3ThreadSafeMalloc( sizeof(*pOpen) );
       if( pOpen==0 ){
         releaseLockInfo(pLock);
         rc = 1;
@@ -583,7 +584,7 @@ static int findLockInfo(
       pOld = sqlite3HashInsert(&openHash, &pOpen->key, sizeof(key2), pOpen);
       if( pOld!=0 ){
         assert( pOld==pOpen );
-        sqliteFree(pOpen);
+        sqlite3ThreadSafeFree(pOpen);
         releaseLockInfo(pLock);
         rc = 1;
         goto exit_findlockinfo;
@@ -1526,7 +1527,7 @@ static int unixClose(OsFile **pId){
   id->isOpen = 0;
   TRACE2("CLOSE   %-3d\n", id->h);
   OpenCounter(-1);
-  sqliteFree(id);
+  sqlite3ThreadSafeFree(id);
   *pId = 0;
   return SQLITE_OK;
 }
@@ -1635,7 +1636,7 @@ static int allocateUnixFile(unixFile *pInit, OsFile **pId){
   pInit->fullSync = 0;
   pInit->locktype = 0;
   SET_THREADID(pInit);
-  pNew = sqliteMalloc( sizeof(unixFile) );
+  pNew = sqlite3ThreadSafeMalloc( sizeof(unixFile) );
   if( pNew==0 ){
     close(pInit->h);
     sqlite3OsEnterMutex();
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 99619bba3..07ba7e942 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.481 2006/02/11 01:25:51 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.482 2006/02/14 10:48:39 danielk1977 Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -267,7 +267,6 @@ extern int sqlite3_nFree;        /* Number of sqliteFree() calls */
 extern int sqlite3_iMallocFail;  /* Fail sqliteMalloc() after this many calls */
 extern int sqlite3_iMallocReset; /* Set iMallocFail to this when it reaches 0 */
 
-
 extern void *sqlite3_pFirst;         /* Pointer to linked list of allocations */
 extern int sqlite3_nMaxAlloc;        /* High water mark of ThreadData.nAlloc */
 extern int sqlite3_mallocDisallowed; /* assert() in sqlite3Malloc() if set */
@@ -276,8 +275,8 @@ extern const char *sqlite3_zFile;    /* Filename to associate debug info with */
 extern int sqlite3_iLine;            /* Line number for debug info */
 
 #define ENTER_MALLOC (sqlite3_zFile = __FILE__, sqlite3_iLine = __LINE__)
-#define sqliteMalloc(x)          (ENTER_MALLOC, sqlite3Malloc(x))
-#define sqliteMallocRaw(x)       (ENTER_MALLOC, sqlite3MallocRaw(x))
+#define sqliteMalloc(x)          (ENTER_MALLOC, sqlite3Malloc(x,1))
+#define sqliteMallocRaw(x)       (ENTER_MALLOC, sqlite3MallocRaw(x,1))
 #define sqliteRealloc(x,y)       (ENTER_MALLOC, sqlite3Realloc(x,y))
 #define sqliteStrDup(x)          (ENTER_MALLOC, sqlite3StrDup(x))
 #define sqliteStrNDup(x,y)       (ENTER_MALLOC, sqlite3StrNDup(x,y))
@@ -1483,8 +1482,8 @@ int sqlite3Compare(const char *, const char *);
 int sqlite3SortCompare(const char *, const char *);
 void sqlite3RealToSortable(double r, char *);
 
-void *sqlite3Malloc(int);
-void *sqlite3MallocRaw(int);
+void *sqlite3Malloc(int,int);
+void *sqlite3MallocRaw(int,int);
 void sqlite3Free(void*);
 void *sqlite3Realloc(void*,int);
 char *sqlite3StrDup(const char*);
@@ -1767,6 +1766,14 @@ void sqlite3AbortOtherActiveVdbes(sqlite3 *, Vdbe *);
   #define sqlite3MallocAllow()
 #endif
 
+#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
+  void *sqlite3ThreadSafeMalloc(int);
+  void sqlite3ThreadSafeFree(void *);
+#else
+  #define sqlite3ThreadSafeMalloc sqlite3MallocX
+  #define sqlite3ThreadSafeFree sqlite3FreeX
+#endif
+
 #ifdef SQLITE_SSE
 #include "sseInt.h"
 #endif
diff --git a/src/test_async.c b/src/test_async.c
index a05ed2a0c..376058356 100644
--- a/src/test_async.c
+++ b/src/test_async.c
@@ -475,6 +475,11 @@ static int asyncRead(OsFile *id, void *obuf, int amt){
   i64 filesize;
   int nRead;
   AsyncFile *pFile = (AsyncFile *)id;
+  OsFile *pBase = pFile->pBaseRead;
+
+  if( !pBase ){
+    pBase = pFile->pBaseWrite;
+  }
 
   /* If an I/O error has previously occurred on this file, then all
   ** subsequent operations fail.
@@ -486,18 +491,18 @@ static int asyncRead(OsFile *id, void *obuf, int amt){
   /* Grab the write queue mutex for the duration of the call */
   pthread_mutex_lock(&async.queueMutex);
 
-  if( pFile->pBaseRead ){
-    rc = sqlite3OsFileSize(pFile->pBaseRead, &filesize);
+  if( pBase ){
+    rc = sqlite3OsFileSize(pBase, &filesize);
     if( rc!=SQLITE_OK ){
       goto asyncread_out;
     }
-    rc = sqlite3OsSeek(pFile->pBaseRead, pFile->iOffset);
+    rc = sqlite3OsSeek(pBase, pFile->iOffset);
     if( rc!=SQLITE_OK ){
       goto asyncread_out;
     }
     nRead = MIN(filesize - pFile->iOffset, amt);
     if( nRead>0 ){
-      rc = sqlite3OsRead(pFile->pBaseRead, obuf, nRead);
+      rc = sqlite3OsRead(pBase, obuf, nRead);
       TRACE(("READ %s %d bytes at %d\n", pFile->zName, nRead, pFile->iOffset));
     }
   }
@@ -562,6 +567,9 @@ int asyncFileSize(OsFile *id, i64 *pSize){
   ** file-system.
   */
   pBase = ((AsyncFile *)id)->pBaseRead;
+  if( !pBase ){
+    pBase = ((AsyncFile *)id)->pBaseWrite;
+  }
   if( pBase ){
     rc = sqlite3OsFileSize(pBase, &s);
   }
@@ -802,8 +810,8 @@ static int asyncFileExists(const char *z){
 */
 static void asyncEnable(int enable){
   if( enable && xOrigOpenReadWrite==0 ){
+    assert(sqlite3Os.xOpenReadWrite);
     sqlite3HashInit(&async.aLock, SQLITE_HASH_BINARY, 1);
-    
     xOrigOpenReadWrite = sqlite3Os.xOpenReadWrite;
     xOrigOpenReadOnly = sqlite3Os.xOpenReadOnly;
     xOrigOpenExclusive = sqlite3Os.xOpenExclusive;
@@ -817,10 +825,11 @@ static void asyncEnable(int enable){
     sqlite3Os.xDelete = asyncDelete;
     sqlite3Os.xFileExists = asyncFileExists;
     sqlite3Os.xSyncDirectory = asyncSyncDirectory;
+    assert(sqlite3Os.xOpenReadWrite);
   }
   if( !enable && xOrigOpenReadWrite!=0 ){
+    assert(sqlite3Os.xOpenReadWrite);
     sqlite3HashClear(&async.aLock);
-
     sqlite3Os.xOpenReadWrite = xOrigOpenReadWrite;
     sqlite3Os.xOpenReadOnly = xOrigOpenReadOnly;
     sqlite3Os.xOpenExclusive = xOrigOpenExclusive;
@@ -834,6 +843,7 @@ static void asyncEnable(int enable){
     xOrigDelete = 0;
     xOrigFileExists = 0;
     xOrigSyncDirectory = 0;
+    assert(sqlite3Os.xOpenReadWrite);
   }
 }
 
@@ -947,8 +957,8 @@ static void *asyncWriterThread(void *NotUsed){
 
       case ASYNC_CLOSE:
         TRACE(("CLOSE %s\n", p->pFile->zName));
-        sqlite3OsClose(&p->pFile->pBaseRead);
         sqlite3OsClose(&p->pFile->pBaseWrite);
+        sqlite3OsClose(&p->pFile->pBaseRead);
         sqlite3OsFree(p->pFile);
         break;
 
@@ -979,12 +989,13 @@ static void *asyncWriterThread(void *NotUsed){
         int delFlag = ((p->iOffset)?1:0);
         OsFile *pBase = 0;
         TRACE(("OPEN %s delFlag=%d\n", p->zBuf, delFlag));
+        assert(pFile->pBaseRead==0 && pFile->pBaseWrite==0);
         rc = xOrigOpenExclusive(p->zBuf, &pBase, delFlag);
         assert( holdingMutex==0 );
         pthread_mutex_lock(&async.queueMutex);
         holdingMutex = 1;
         if( rc==SQLITE_OK ){
-          pFile->pBaseRead = pBase;
+          pFile->pBaseWrite = pBase;
         }
         break;
       }
@@ -1173,6 +1184,7 @@ static int testAsyncWait(
     Tcl_AppendResult(interp, "would block forever", (char*)0);
     return TCL_ERROR;
   }
+
   while( cnt-- && !pthread_mutex_trylock(&async.writerMutex) ){
     pthread_mutex_unlock(&async.writerMutex);
     sched_yield();
diff --git a/src/util.c b/src/util.c
index 0f2bc94aa..13a084e83 100644
--- a/src/util.c
+++ b/src/util.c
@@ -14,7 +14,7 @@
 ** This file contains functions for allocating memory, comparing
 ** strings, and stuff like that.
 **
-** $Id: util.c,v 1.184 2006/02/06 21:22:31 drh Exp $
+** $Id: util.c,v 1.185 2006/02/14 10:48:39 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -443,6 +443,7 @@ int sqlite3OutstandingMallocs(Tcl_Interp *interp){
 ** This is the test layer's wrapper around sqlite3OsMalloc().
 */
 static void * OSMALLOC(int n){
+  sqlite3OsEnterMutex();
 #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
   sqlite3_nMaxAlloc = 
       MAX(sqlite3_nMaxAlloc, sqlite3ThreadDataReadOnly()->nAlloc);
@@ -455,8 +456,10 @@ static void * OSMALLOC(int n){
     sqlite3_nMalloc++;
     applyGuards(p);
     linkAlloc(p);
+    sqlite3OsLeaveMutex();
     return (void *)(&p[TESTALLOC_NGUARD + 2*sizeof(void *)/sizeof(u32)]);
   }
+  sqlite3OsLeaveMutex();
   return 0;
 }
 
@@ -473,12 +476,14 @@ static int OSSIZEOF(void *p){
 ** pointer to the space allocated for the application to use.
 */
 static void OSFREE(void *pFree){
+  sqlite3OsEnterMutex();
   u32 *p = (u32 *)getOsPointer(pFree);   /* p points to Os level allocation */
   checkGuards(p);
   unlinkAlloc(p);
   memset(pFree, 0x55, OSSIZEOF(pFree));
   sqlite3OsFree(p);
   sqlite3_nFree++;
+  sqlite3OsLeaveMutex();
 }
 
 /*
@@ -578,14 +583,14 @@ static void updateMemoryUsedCount(int n){
 ** sqlite3OsMalloc(). If the Malloc() call fails, attempt to free memory 
 ** by calling sqlite3_release_memory().
 */
-void *sqlite3MallocRaw(int n){
+void *sqlite3MallocRaw(int n, int doMemManage){
   void *p = 0;
-  if( n>0 && !sqlite3MallocFailed() && enforceSoftLimit(n) ){
+  if( n>0 && !sqlite3MallocFailed() && (!doMemManage || enforceSoftLimit(n)) ){
     while( (p = OSMALLOC(n))==0 && sqlite3_release_memory(n) );
     if( !p ){
       sqlite3FailedMalloc();
       OSMALLOC_FAILED();
-    }else{
+    }else if( doMemManage ){
       updateMemoryUsedCount(OSSIZEOF(p));
     }
   }
@@ -603,7 +608,7 @@ void *sqlite3Realloc(void *p, int n){
   }
 
   if( !p ){
-    return sqlite3Malloc(n);
+    return sqlite3Malloc(n, 1);
   }else{
     void *np = 0;
 #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
@@ -648,8 +653,8 @@ void *sqlite3MallocX(int n){
 ** These two are implemented as wrappers around sqlite3MallocRaw(), 
 ** sqlite3Realloc() and sqlite3Free().
 */ 
-void *sqlite3Malloc(int n){
-  void *p = sqlite3MallocRaw(n);
+void *sqlite3Malloc(int n, int doMemManage){
+  void *p = sqlite3MallocRaw(n, doMemManage);
   if( p ){
     memset(p, 0, n);
   }
@@ -664,6 +669,33 @@ void sqlite3ReallocOrFree(void **pp, int n){
 }
 
 /*
+** sqlite3ThreadSafeMalloc() and sqlite3ThreadSafeFree() are used in those
+** rare scenarios where sqlite may allocate memory in one thread and free
+** it in another. They are exactly the same as sqlite3Malloc() and 
+** sqlite3Free() except that:
+**
+**   * The allocated memory is not included in any calculations with 
+**     respect to the soft-heap-limit, and
+**
+**   * sqlite3ThreadSafeMalloc() must be matched with ThreadSafeFree(),
+**     not sqlite3Free(). Calling sqlite3Free() on memory obtained from
+**     ThreadSafeMalloc() will cause an error somewhere down the line.
+*/
+#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
+void *sqlite3ThreadSafeMalloc(int n){
+  ENTER_MALLOC;
+  return sqlite3Malloc(n, 0);
+}
+void sqlite3ThreadSafeFree(void *p){
+  ENTER_MALLOC;
+  if( p ){
+    OSFREE(p);
+  }
+}
+#endif
+
+
+/*
 ** Return the number of bytes allocated at location p. p must be either 
 ** a NULL pointer (in which case 0 is returned) or a pointer returned by 
 ** sqlite3Malloc(), sqlite3Realloc() or sqlite3ReallocOrFree().
@@ -689,14 +721,14 @@ int sqlite3AllocSize(void *p){
 char *sqlite3StrDup(const char *z){
   char *zNew;
   if( z==0 ) return 0;
-  zNew = sqlite3MallocRaw(strlen(z)+1);
+  zNew = sqlite3MallocRaw(strlen(z)+1, 1);
   if( zNew ) strcpy(zNew, z);
   return zNew;
 }
 char *sqlite3StrNDup(const char *z, int n){
   char *zNew;
   if( z==0 ) return 0;
-  zNew = sqlite3MallocRaw(n+1);
+  zNew = sqlite3MallocRaw(n+1, 1);
   if( zNew ){
     memcpy(zNew, z, n);
     zNew[n] = 0;