Add the memory fault simulator to mem5.c. Enable soft heap limit on mem5.c.

Limit the size of hash tables and the vdbefifo when using mem5.c. (CVS 4795)

FossilOrigin-Name: 63da5d97542e4f54c33329833477c8d96ce05dd0

9 files changed, 123 insertions, 52 deletions

diff --git a/src/hash.c b/src/hash.c
index 9a91e85e2..0664f6dbc 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -12,7 +12,7 @@
 ** This is the implementation of generic hash-tables
 ** used in SQLite.
 **
-** $Id: hash.c,v 1.25 2008/01/22 21:30:53 drh Exp $
+** $Id: hash.c,v 1.26 2008/02/18 22:24:58 drh Exp $
 */
 #include "sqliteInt.h"
 #include <assert.h>
@@ -221,7 +221,12 @@ static void rehash(Hash *pH, int new_size){
   HashElem *elem, *next_elem;    /* For looping over existing elements */
   int (*xHash)(const void*,int); /* The hash function */
 
-  assert( (new_size & (new_size-1))==0 );
+#ifdef SQLITE_MALLOC_SOFT_LIMIT
+  if( new_size*sizeof(struct _ht)>SQLITE_MALLOC_SOFT_LIMIT ){
+    new_size = SQLITE_MALLOC_SOFT_LIMIT/sizeof(struct _ht);
+  }
+  if( new_size==pH->htsize ) return;
+#endif
 
   /* There is a call to sqlite3_malloc() inside rehash(). If there is
   ** already an allocation at pH->ht, then if this malloc() fails it
@@ -324,8 +329,7 @@ HashElem *sqlite3HashFindElem(const Hash *pH, const void *pKey, int nKey){
   xHash = hashFunction(pH->keyClass);
   assert( xHash!=0 );
   h = (*xHash)(pKey,nKey);
-  assert( (pH->htsize & (pH->htsize-1))==0 );
-  elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
+  elem = findElementGivenHash(pH,pKey,nKey, h % pH->htsize);
   return elem;
 }
 
@@ -365,21 +369,22 @@ void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
   xHash = hashFunction(pH->keyClass);
   assert( xHash!=0 );
   hraw = (*xHash)(pKey, nKey);
-  assert( (pH->htsize & (pH->htsize-1))==0 );
-  h = hraw & (pH->htsize-1);
-  elem = findElementGivenHash(pH,pKey,nKey,h);
-  if( elem ){
-    void *old_data = elem->data;
-    if( data==0 ){
-      removeElementGivenHash(pH,elem,h);
-    }else{
-      elem->data = data;
-      if( !pH->copyKey ){
-        elem->pKey = (void *)pKey;
+  if( pH->htsize ){
+    h = hraw % pH->htsize;
+    elem = findElementGivenHash(pH,pKey,nKey,h);
+    if( elem ){
+      void *old_data = elem->data;
+      if( data==0 ){
+        removeElementGivenHash(pH,elem,h);
+      }else{
+        elem->data = data;
+        if( !pH->copyKey ){
+          elem->pKey = (void *)pKey;
+        }
+        assert(nKey==elem->nKey);
       }
-      assert(nKey==elem->nKey);
+      return old_data;
     }
-    return old_data;
   }
   if( data==0 ) return 0;
   new_elem = (HashElem*)sqlite3_malloc( sizeof(HashElem) );
@@ -397,7 +402,7 @@ void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
   new_elem->nKey = nKey;
   pH->count++;
   if( pH->htsize==0 ){
-    rehash(pH,8);
+    rehash(pH, 128/sizeof(pH->ht[0]));
     if( pH->htsize==0 ){
       pH->count = 0;
       if( pH->copyKey ){
@@ -411,8 +416,7 @@ void *sqlite3HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
     rehash(pH,pH->htsize*2);
   }
   assert( pH->htsize>0 );
-  assert( (pH->htsize & (pH->htsize-1))==0 );
-  h = hraw & (pH->htsize-1);
+  h = hraw % pH->htsize;
   insertElement(pH, &pH->ht[h], new_elem);
   new_elem->data = data;
   return 0;
diff --git a/src/mem5.c b/src/mem5.c
index abc0837d9..e73b13aae 100644
--- a/src/mem5.c
+++ b/src/mem5.c
@@ -20,7 +20,7 @@
 ** This version of the memory allocation subsystem is used if
 ** and only if SQLITE_POW2_MEMORY_SIZE is defined.
 **
-** $Id: mem5.c,v 1.2 2008/02/16 16:21:46 drh Exp $
+** $Id: mem5.c,v 1.3 2008/02/18 22:24:58 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -93,8 +93,15 @@ struct Mem5Block {
 */
 static struct {
   /*
-  ** True if we are evaluating an out-of-memory callback.
+  ** The alarm callback and its arguments.  The mem.mutex lock will
+  ** be held while the callback is running.  Recursive calls into
+  ** the memory subsystem are allowed, but no new callbacks will be
+  ** issued.  The alarmBusy variable is set to prevent recursive
+  ** callbacks.
   */
+  sqlite3_int64 alarmThreshold;
+  void (*alarmCallback)(void*, sqlite3_int64,int);
+  void *alarmArg;
   int alarmBusy;
   
   /*
@@ -221,6 +228,25 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){
   return n;
 }
 
+
+/*
+** Trigger the alarm 
+*/
+static void memsys5Alarm(int nByte){
+  void (*xCallback)(void*,sqlite3_int64,int);
+  sqlite3_int64 nowUsed;
+  void *pArg;
+  if( mem.alarmCallback==0 || mem.alarmBusy  ) return;
+  mem.alarmBusy = 1;
+  xCallback = mem.alarmCallback;
+  nowUsed = mem.currentOut;
+  pArg = mem.alarmArg;
+  sqlite3_mutex_leave(mem.mutex);
+  xCallback(pArg, nowUsed, nByte);
+  sqlite3_mutex_enter(mem.mutex);
+  mem.alarmBusy = 0;
+}
+
 /*
 ** Change the alarm callback.
 **
@@ -234,24 +260,15 @@ int sqlite3_memory_alarm(
   void *pArg,
   sqlite3_int64 iThreshold
 ){
+  memsys5Enter();
+  mem.alarmCallback = xCallback;
+  mem.alarmArg = pArg;
+  mem.alarmThreshold = iThreshold;
+  sqlite3_mutex_leave(mem.mutex);
   return SQLITE_OK;
 }
 
 /*
-** Called when we are unable to satisfy an allocation of nBytes.
-*/
-static void memsys5OutOfMemory(int nByte){
-  if( !mem.alarmBusy ){
-    mem.alarmBusy = 1;
-    assert( sqlite3_mutex_held(mem.mutex) );
-    sqlite3_mutex_leave(mem.mutex);
-    sqlite3_release_memory(nByte);
-    sqlite3_mutex_enter(mem.mutex);
-    mem.alarmBusy = 0;
-  }
-}
-
-/*
 ** Return the size of an outstanding allocation, in bytes.  The
 ** size returned omits the 8-byte header overhead.  This only
 ** works for chunks that are currently checked out.
@@ -296,10 +313,30 @@ static void *memsys5Malloc(int nByte){
   int iLogsize;    /* Log2 of iFullSz/POW2_MIN */
 
   assert( sqlite3_mutex_held(mem.mutex) );
-  if( nByte>mem.maxRequest ) mem.maxRequest = nByte;
+
+  /* Keep track of the maximum allocation request.  Even unfulfilled
+  ** requests are counted */
+  if( nByte>mem.maxRequest ){
+    mem.maxRequest = nByte;
+  }
+
+  /* Simulate a memory allocation fault */
+  if( sqlite3FaultStep(SQLITE_FAULTINJECTOR_MALLOC) ) return 0;
+
+  /* Round nByte up to the next valid power of two */
   if( nByte>POW2_MAX ) return 0;
   for(iFullSz=POW2_MIN, iLogsize=0; iFullSz<nByte; iFullSz *= 2, iLogsize++){}
 
+  /* If we will be over the memory alarm threshold after this allocation,
+  ** then trigger the memory overflow alarm */
+  if( mem.alarmCallback!=0 && mem.currentOut+iFullSz>=mem.alarmThreshold ){
+    memsys5Alarm(iFullSz);
+  }
+
+  /* Make sure mem.aiFreelist[iLogsize] contains at least one free
+  ** block.  If not, then split a block of the next larger power of
+  ** two in order to create a new free block of size iLogsize.
+  */
   for(iBin=iLogsize; mem.aiFreelist[iBin]<0 && iBin<NSIZE; iBin++){}
   if( iBin>=NSIZE ) return 0;
   i = memsys5UnlinkFirst(iBin);
@@ -313,6 +350,7 @@ static void *memsys5Malloc(int nByte){
   }
   mem.aCtrl[i] = iLogsize;
 
+  /* Update allocator performance statistics. */
   mem.nAlloc++;
   mem.totalAlloc += iFullSz;
   mem.totalExcess += iFullSz - nByte;
@@ -321,6 +359,7 @@ static void *memsys5Malloc(int nByte){
   if( mem.maxCount<mem.currentCount ) mem.maxCount = mem.currentCount;
   if( mem.maxOut<mem.currentOut ) mem.maxOut = mem.currentOut;
 
+  /* Return a pointer to the allocated memory. */
   return (void*)&mem.aPool[i];
 }
 
diff --git a/src/pager.c b/src/pager.c
index 80469db9f..3e72e84a5 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -18,7 +18,7 @@
 ** file simultaneously, or one process from reading the database while
 ** another is writing.
 **
-** @(#) $Id: pager.c,v 1.407 2008/02/18 14:47:34 drh Exp $
+** @(#) $Id: pager.c,v 1.408 2008/02/18 22:24:58 drh Exp $
 */
 #ifndef SQLITE_OMIT_DISKIO
 #include "sqliteInt.h"
@@ -677,6 +677,12 @@ static int pageInStatement(PgHdr *pPg){
 static void pager_resize_hash_table(Pager *pPager, int N){
   PgHdr **aHash, *pPg;
   assert( N>0 && (N&(N-1))==0 );
+#ifdef SQLITE_MALLOC_SOFT_LIMIT
+  if( N*sizeof(aHash[0])>SQLITE_MALLOC_SOFT_LIMIT ){
+    N = SQLITE_MALLOC_SOFT_LIMIT/sizeof(aHash[0]);
+  }
+  if( N==pPager->nHash ) return;
+#endif
   pagerLeave(pPager);
   sqlite3FaultBenign(SQLITE_FAULTINJECTOR_MALLOC, pPager->aHash!=0);
   aHash = sqlite3MallocZero( sizeof(aHash[0])*N );
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index e81802ef1..3de750c36 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.663 2008/02/18 14:47:34 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.664 2008/02/18 22:24:58 drh Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -144,6 +144,14 @@
 #endif
 
 /*
+** If SQLITE_MALLOC_SOFT_LIMIT is defined, then try to keep the
+** sizes of memory allocations below this value where possible.
+*/
+#if defined(SQLITE_POW2_MEMORY_SIZE) && !defined(SQLITE_MALLOC_SOFT_LIMIT)
+# define SQLITE_MALLOC_SOFT_LIMIT 1024
+#endif
+
+/*
 ** We need to define _XOPEN_SOURCE as follows in order to enable
 ** recursive mutexes on most unix systems.  But Mac OS X is different.
 ** The _XOPEN_SOURCE define causes problems for Mac OS X we are told,
@@ -403,6 +411,7 @@ typedef struct WhereLevel WhereLevel;
 #include "os.h"
 #include "mutex.h"
 
+
 /*
 ** Each database file to be accessed by the system is an instance
 ** of the following structure.  There are normally two of these structures
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index ebb8c0f7c..4f1a6c928 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -12,7 +12,7 @@
 ** A TCL Interface to SQLite.  Append this file to sqlite3.c and
 ** compile the whole thing to build a TCL-enabled version of SQLite.
 **
-** $Id: tclsqlite.c,v 1.208 2008/02/13 18:25:27 danielk1977 Exp $
+** $Id: tclsqlite.c,v 1.209 2008/02/18 22:24:58 drh Exp $
 */
 #include "tcl.h"
 #include <errno.h>
@@ -1702,9 +1702,9 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){
           switch( sqlite3_column_type(pStmt, i) ){
             case SQLITE_BLOB: {
               int bytes = sqlite3_column_bytes(pStmt, i);
-              char *zBlob = sqlite3_column_blob(pStmt, i);
+              const char *zBlob = sqlite3_column_blob(pStmt, i);
               if( !zBlob ) bytes = 0;
-              pVal = Tcl_NewByteArrayObj(zBlob, bytes);
+              pVal = Tcl_NewByteArrayObj((u8*)zBlob, bytes);
               break;
             }
             case SQLITE_INTEGER: {
diff --git a/src/test1.c b/src/test1.c
index adb979781..4cc319854 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -13,7 +13,7 @@
 ** is not included in the SQLite library.  It is used for automated
 ** testing of the SQLite library.
 **
-** $Id: test1.c,v 1.288 2008/02/13 18:25:27 danielk1977 Exp $
+** $Id: test1.c,v 1.289 2008/02/18 22:24:58 drh Exp $
 */
 #include "sqliteInt.h"
 #include "tcl.h"
@@ -954,7 +954,7 @@ static int test_create_function(
   /* Use the sqlite3_create_function16() API here. Mainly for fun, but also 
   ** because it is not tested anywhere else. */
   if( rc==SQLITE_OK ){
-    void *zUtf16;
+    const void *zUtf16;
     sqlite3_value *pVal;
     sqlite3_mutex_enter(db->mutex);
     pVal = sqlite3ValueNew(db);
@@ -2153,7 +2153,7 @@ static int test_collate(
   rc = sqlite3_create_collation(db, "test_collate", SQLITE_UTF8, 
           (void *)SQLITE_UTF8, val?test_collate_func:0);
   if( rc==SQLITE_OK ){
-    void *zUtf16;
+    const void *zUtf16;
     if( TCL_OK!=Tcl_GetBooleanFromObj(interp, objv[3], &val) ) return TCL_ERROR;
     rc = sqlite3_create_collation(db, "test_collate", SQLITE_UTF16LE, 
             (void *)SQLITE_UTF16LE, val?test_collate_func:0);
diff --git a/src/test_malloc.c b/src/test_malloc.c
index ef7512e64..642e5c7d0 100644
--- a/src/test_malloc.c
+++ b/src/test_malloc.c
@@ -13,7 +13,7 @@
 ** This file contains code used to implement test interfaces to the
 ** memory allocation subsystem.
 **
-** $Id: test_malloc.c,v 1.13 2008/02/16 16:21:46 drh Exp $
+** $Id: test_malloc.c,v 1.14 2008/02/18 22:24:58 drh Exp $
 */
 #include "sqliteInt.h"
 #include "tcl.h"
@@ -468,7 +468,7 @@ static int test_memdebug_pending(
     return TCL_ERROR;
   }
 
-#ifdef SQLITE_MEMDEBUG
+#if defined(SQLITE_MEMDEBUG) || defined(SQLITE_POW2_MEMORY_SIZE)
   {
     int nPending = sqlite3_test_control(SQLITE_TESTCTRL_FAULT_PENDING,
                                         SQLITE_FAULTINJECTOR_MALLOC);
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 9e6273c39..35a236445 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -142,7 +142,7 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
   i = p->nOp;
   assert( p->magic==VDBE_MAGIC_INIT );
   if( p->nOpAlloc<=i ){
-    resizeOpArray(p, p->nOpAlloc*2 + 100);
+    resizeOpArray(p, p->nOpAlloc ? p->nOpAlloc*2 : 1024/sizeof(Op));
     if( p->db->mallocFailed ){
       return 0;
     }
@@ -335,7 +335,8 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp){
   int addr;
   assert( p->magic==VDBE_MAGIC_INIT );
   if( p->nOp + nOp > p->nOpAlloc ){
-    resizeOpArray(p, p->nOp*2 + nOp);
+    resizeOpArray(p, p->nOpAlloc ? p->nOpAlloc*2 : 1024/sizeof(Op));
+    assert( p->nOp+nOp<=p->nOpAlloc || p->db->mallocFailed );
   }
   if( p->db->mallocFailed ){
     return 0;
diff --git a/src/vdbefifo.c b/src/vdbefifo.c
index f03f5c6e9..782ac99c7 100644
--- a/src/vdbefifo.c
+++ b/src/vdbefifo.c
@@ -16,13 +16,25 @@
 #include "vdbeInt.h"
 
 /*
+** Constants FIFOSIZE_FIRST and FIFOSIZE_MAX are the initial
+** number of entries in a fifo page and the maximum number of
+** entries in a fifo page.
+*/
+#define FIFOSIZE_FIRST (((128-sizeof(FifoPage))/8)+1)
+#ifdef SQLITE_MALLOC_SOFT_LIMIT
+# define FIFOSIZE_MAX   (((SQLITE_MALLOC_SOFT_LIMIT-sizeof(FifoPage))/8)+1)
+#else
+# define FIFOSIZE_MAX   (((262144-sizeof(FifoPage))/8)+1)
+#endif
+
+/*
 ** Allocate a new FifoPage and return a pointer to it.  Return NULL if
 ** we run out of memory.  Leave space on the page for nEntry entries.
 */
 static FifoPage *allocateFifoPage(int nEntry){
   FifoPage *pPage;
-  if( nEntry>32767 ){
-    nEntry = 32767;
+  if( nEntry>FIFOSIZE_MAX ){
+    nEntry = FIFOSIZE_MAX;
   }
   pPage = sqlite3_malloc( sizeof(FifoPage) + sizeof(i64)*(nEntry-1) );
   if( pPage ){
@@ -50,7 +62,7 @@ int sqlite3VdbeFifoPush(Fifo *pFifo, i64 val){
   FifoPage *pPage;
   pPage = pFifo->pLast;
   if( pPage==0 ){
-    pPage = pFifo->pLast = pFifo->pFirst = allocateFifoPage(20);
+    pPage = pFifo->pLast = pFifo->pFirst = allocateFifoPage(FIFOSIZE_FIRST);
     if( pPage==0 ){
       return SQLITE_NOMEM;
     }