Merge all changes in the latest 3.8.11 beta into the sessions branch.

Changes include the rename of OTA to RBU, the WITHOUT-ROWID-OR-Skipscan fix,
and improvements to pcache1.

FossilOrigin-Name: 7f0ee77062d2fcb014942c7c62c163ccc801f21b

15 files changed, 115 insertions, 75 deletions

diff --git a/src/btree.c b/src/btree.c
index 17c7bb67c..2692ade8e 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -8959,7 +8959,7 @@ static int checkTreePage(
   const char *saved_zPfx = pCheck->zPfx;
   int saved_v1 = pCheck->v1;
   int saved_v2 = pCheck->v2;
-  u8 savedIsInit;
+  u8 savedIsInit = 0;
 
   /* Check that the page exists
   */
diff --git a/src/date.c b/src/date.c
index 5f3f247ca..6b11d9904 100644
--- a/src/date.c
+++ b/src/date.c
@@ -355,7 +355,7 @@ static void computeYMD(DateTime *p){
     A = Z + 1 + A - (A/4);
     B = A + 1524;
     C = (int)((B - 122.1)/365.25);
-    D = (36525*C)/100;
+    D = (36525*(C&32767))/100;
     E = (int)((B-D)/30.6001);
     X1 = (int)(30.6001*E);
     p->D = B - D - X1;
diff --git a/src/main.c b/src/main.c
index addb3987f..d0545ccf1 100644
--- a/src/main.c
+++ b/src/main.c
@@ -642,6 +642,7 @@ int sqlite3_config(int op, ...){
 ** the lookaside memory.
 */
 static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
+#ifndef SQLITE_OMIT_LOOKASIDE
   void *pStart;
   if( db->lookaside.nOut ){
     return SQLITE_BUSY;
@@ -692,6 +693,7 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
     db->lookaside.bEnabled = 0;
     db->lookaside.bMalloced = 0;
   }
+#endif /* SQLITE_OMIT_LOOKASIDE */
   return SQLITE_OK;
 }
 
diff --git a/src/pcache1.c b/src/pcache1.c
index a345e6760..187f09f59 100644
--- a/src/pcache1.c
+++ b/src/pcache1.c
@@ -191,6 +191,7 @@ static SQLITE_WSD struct PCacheGlobal {
   */
   int isInit;                    /* True if initialized */
   int separateCache;             /* Use a new PGroup for each PCache */
+  int nInitPage;                 /* Initial bulk allocation size */   
   int szSlot;                    /* Size of each free slot */
   int nSlot;                     /* The number of pcache slots */
   int nReserve;                  /* Try to keep nFreeSlot above this */
@@ -260,6 +261,43 @@ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){
 }
 
 /*
+** Try to initialize the pCache->pFree and pCache->pBulk fields.  Return
+** true if pCache->pFree ends up containing one or more free pages.
+*/
+static int pcache1InitBulk(PCache1 *pCache){
+  i64 szBulk;
+  char *zBulk;
+  if( pcache1.nInitPage==0 ) return 0;
+  /* Do not bother with a bulk allocation if the cache size very small */
+  if( pCache->nMax<3 ) return 0;
+  sqlite3BeginBenignMalloc();
+  if( pcache1.nInitPage>0 ){
+    szBulk = pCache->szAlloc * (i64)pcache1.nInitPage;
+  }else{
+    szBulk = -1024 * (i64)pcache1.nInitPage;
+  }
+  if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){
+    szBulk = pCache->szAlloc*pCache->nMax;
+  }
+  zBulk = pCache->pBulk = sqlite3Malloc( szBulk );
+  sqlite3EndBenignMalloc();
+  if( zBulk ){
+    int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc;
+    int i;
+    for(i=0; i<nBulk; i++){
+      PgHdr1 *pX = (PgHdr1*)&zBulk[pCache->szPage];
+      pX->page.pBuf = zBulk;
+      pX->page.pExtra = &pX[1];
+      pX->isBulkLocal = 1;
+      pX->pNext = pCache->pFree;
+      pCache->pFree = pX;
+      zBulk += pCache->szAlloc;
+    }
+  }
+  return pCache->pFree!=0;
+}
+
+/*
 ** Malloc function used within this file to allocate space from the buffer
 ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no 
 ** such buffer exists or there is no space left in it, this function falls 
@@ -359,7 +397,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){
   void *pPg;
 
   assert( sqlite3_mutex_held(pCache->pGroup->mutex) );
-  if( pCache->pFree ){
+  if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){
     p = pCache->pFree;
     pCache->pFree = p->pNext;
     p->pNext = 0;
@@ -563,7 +601,8 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){
 ** If there are currently more than nMaxPage pages allocated, try
 ** to recycle pages to reduce the number allocated to nMaxPage.
 */
-static void pcache1EnforceMaxPage(PGroup *pGroup){
+static void pcache1EnforceMaxPage(PCache1 *pCache){
+  PGroup *pGroup = pCache->pGroup;
   assert( sqlite3_mutex_held(pGroup->mutex) );
   while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){
     PgHdr1 *p = pGroup->pLruTail;
@@ -572,6 +611,10 @@ static void pcache1EnforceMaxPage(PGroup *pGroup){
     pcache1PinPage(p);
     pcache1RemoveFromHash(p, 1);
   }
+  if( pCache->nPage==0 && pCache->pBulk ){
+    sqlite3_free(pCache->pBulk);
+    pCache->pBulk = pCache->pFree = 0;
+  }
 }
 
 /*
@@ -647,6 +690,14 @@ static int pcache1Init(void *NotUsed){
     pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM);
   }
 #endif
+  if( pcache1.separateCache
+   && sqlite3GlobalConfig.nPage!=0
+   && sqlite3GlobalConfig.pPage==0
+  ){
+    pcache1.nInitPage = sqlite3GlobalConfig.nPage;
+  }else{
+    pcache1.nInitPage = 0;
+  }
   pcache1.grp.mxPinned = 10;
   pcache1.isInit = 1;
   return SQLITE_OK;
@@ -701,36 +752,6 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){
       pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
     }
     pcache1LeaveMutex(pGroup);
-    /* Try to initialize the local bulk pagecache line allocation if using
-    ** separate caches and if nPage!=0 */
-    if( pcache1.separateCache
-     && sqlite3GlobalConfig.nPage!=0
-     && sqlite3GlobalConfig.pPage==0
-    ){
-      int szBulk;
-      char *zBulk;
-      sqlite3BeginBenignMalloc();
-      if( sqlite3GlobalConfig.nPage>0 ){
-        szBulk = pCache->szAlloc * sqlite3GlobalConfig.nPage;
-      }else{
-        szBulk = -1024*sqlite3GlobalConfig.nPage;
-      }
-      zBulk = pCache->pBulk = sqlite3Malloc( szBulk );
-      sqlite3EndBenignMalloc();
-      if( zBulk ){
-        int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc;
-        int i;
-        for(i=0; i<nBulk; i++){
-          PgHdr1 *pX = (PgHdr1*)&zBulk[szPage];
-          pX->page.pBuf = zBulk;
-          pX->page.pExtra = &pX[1];
-          pX->isBulkLocal = 1;
-          pX->pNext = pCache->pFree;
-          pCache->pFree = pX;
-          zBulk += pCache->szAlloc;
-        }
-      }
-    }
     if( pCache->nHash==0 ){
       pcache1Destroy((sqlite3_pcache*)pCache);
       pCache = 0;
@@ -753,7 +774,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){
     pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
     pCache->nMax = nMax;
     pCache->n90pct = pCache->nMax*9/10;
-    pcache1EnforceMaxPage(pGroup);
+    pcache1EnforceMaxPage(pCache);
     pcache1LeaveMutex(pGroup);
   }
 }
@@ -771,7 +792,7 @@ static void pcache1Shrink(sqlite3_pcache *p){
     pcache1EnterMutex(pGroup);
     savedMaxPage = pGroup->nMaxPage;
     pGroup->nMaxPage = 0;
-    pcache1EnforceMaxPage(pGroup);
+    pcache1EnforceMaxPage(pCache);
     pGroup->nMaxPage = savedMaxPage;
     pcache1LeaveMutex(pGroup);
   }
@@ -1108,7 +1129,7 @@ static void pcache1Destroy(sqlite3_pcache *p){
   assert( pGroup->nMinPage >= pCache->nMin );
   pGroup->nMinPage -= pCache->nMin;
   pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage;
-  pcache1EnforceMaxPage(pGroup);
+  pcache1EnforceMaxPage(pCache);
   pcache1LeaveMutex(pGroup);
   sqlite3_free(pCache->pBulk);
   sqlite3_free(pCache->apHash);
diff --git a/src/pragma.c b/src/pragma.c
index 1b4213844..ffa068572 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -721,6 +721,7 @@ void sqlite3Pragma(
   case PragTyp_CACHE_SIZE: {
     assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
     if( !zRight ){
+      if( sqlite3ReadSchema(pParse) ) goto pragma_out;
       returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size);
     }else{
       int size = sqlite3Atoi(zRight);
diff --git a/src/pragma.h b/src/pragma.h
index bbf141ee2..9e206dac4 100644
--- a/src/pragma.h
+++ b/src/pragma.h
@@ -86,7 +86,7 @@ static const struct sPragmaNames {
 #if !defined(SQLITE_OMIT_PAGER_PRAGMAS)
   { /* zName:     */ "cache_size",
     /* ePragTyp:  */ PragTyp_CACHE_SIZE,
-    /* ePragFlag: */ PragFlag_NeedSchema,
+    /* ePragFlag: */ 0,
     /* iArg:      */ 0 },
 #endif
 #if !defined(SQLITE_OMIT_FLAG_PRAGMAS)
diff --git a/src/printf.c b/src/printf.c
index 72b9497d7..018df412f 100644
--- a/src/printf.c
+++ b/src/printf.c
@@ -1012,6 +1012,11 @@ char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){
 ** sqlite3_log() must render into a static buffer.  It cannot dynamically
 ** allocate memory because it might be called while the memory allocator
 ** mutex is held.
+**
+** sqlite3VXPrintf() might ask for *temporary* memory allocations for
+** certain format characters (%q) or for very large precisions or widths.
+** Care must be taken that any sqlite3_log() calls that occur while the
+** memory mutex is held do not use these mechanisms.
 */
 static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){
   StrAccum acc;                          /* String accumulator */
diff --git a/src/select.c b/src/select.c
index 13de02942..8ac98f175 100644
--- a/src/select.c
+++ b/src/select.c
@@ -1082,7 +1082,6 @@ static KeyInfo *keyInfoFromExprList(
   return pInfo;
 }
 
-#ifndef SQLITE_OMIT_COMPOUND_SELECT
 /*
 ** Name of the connection operator, used for error messages.
 */
@@ -1096,7 +1095,6 @@ static const char *selectOpName(int id){
   }
   return z;
 }
-#endif /* SQLITE_OMIT_COMPOUND_SELECT */
 
 #ifndef SQLITE_OMIT_EXPLAIN
 /*
@@ -2100,19 +2098,6 @@ static int multiSelectOrderBy(
 );
 
 /*
-** Error message for when two or more terms of a compound select have different
-** size result sets.
-*/
-void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){
-  if( p->selFlags & SF_Values ){
-    sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms");
-  }else{
-    sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s"
-      " do not have the same number of result columns", selectOpName(p->op));
-  }
-}
-
-/*
 ** Handle the special case of a compound-select that originates from a
 ** VALUES clause.  By handling this as a special case, we avoid deep
 ** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT
@@ -2539,6 +2524,19 @@ multi_select_end:
 #endif /* SQLITE_OMIT_COMPOUND_SELECT */
 
 /*
+** Error message for when two or more terms of a compound select have different
+** size result sets.
+*/
+void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){
+  if( p->selFlags & SF_Values ){
+    sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms");
+  }else{
+    sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s"
+      " do not have the same number of result columns", selectOpName(p->op));
+  }
+}
+
+/*
 ** Code an output subroutine for a coroutine implementation of a
 ** SELECT statment.
 **
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index fcae1b147..3ddd8ece5 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -967,9 +967,9 @@ struct sqlite3_io_methods {
 ** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other
 ** VFS should return SQLITE_NOTFOUND for this opcode.
 **
-** <li>[[SQLITE_FCNTL_OTA]]
-** The [SQLITE_FCNTL_OTA] opcode is implemented by the special VFS used by
-** the OTA extension only.  All other VFS should return SQLITE_NOTFOUND for
+** <li>[[SQLITE_FCNTL_RBU]]
+** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by
+** the RBU extension only.  All other VFS should return SQLITE_NOTFOUND for
 ** this opcode.  
 ** </ul>
 */
@@ -997,7 +997,7 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_WIN32_SET_HANDLE       23
 #define SQLITE_FCNTL_WAL_BLOCK              24
 #define SQLITE_FCNTL_ZIPVFS                 25
-#define SQLITE_FCNTL_OTA                    26
+#define SQLITE_FCNTL_RBU                    26
 
 /* deprecated names */
 #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 6edb885c1..7d99d772f 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -189,7 +189,7 @@
 ** Make sure that the compiler intrinsics we desire are enabled when
 ** compiling with an appropriate version of MSVC.
 */
-#if defined(_MSC_VER) && _MSC_VER>=1300
+#if defined(_MSC_VER) && _MSC_VER>=1300 && !defined(_WIN32_WCE)
 #  include <intrin.h>
 #  pragma intrinsic(_byteswap_ushort)
 #  pragma intrinsic(_byteswap_ulong)
@@ -754,7 +754,9 @@ extern const int sqlite3one;
 # if defined(__linux__) \
   || defined(_WIN32) \
   || (defined(__APPLE__) && defined(__MACH__)) \
-  || defined(__sun)
+  || defined(__sun) \
+  || defined(__FreeBSD__) \
+  || defined(__DragonFly__)
 #   define SQLITE_MAX_MMAP_SIZE 0x7fff0000  /* 2147418112 */
 # else
 #   define SQLITE_MAX_MMAP_SIZE 0
diff --git a/src/tclsqlite.c b/src/tclsqlite.c
index 0f572bd01..608b3b2ba 100644
--- a/src/tclsqlite.c
+++ b/src/tclsqlite.c
@@ -3910,7 +3910,7 @@ static void init_all(Tcl_Interp *interp){
     extern int TestSession_Init(Tcl_Interp*);
 #endif
     extern int Fts5tcl_Init(Tcl_Interp *);
-    extern int SqliteOta_Init(Tcl_Interp*);
+    extern int SqliteRbu_Init(Tcl_Interp*);
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
     extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
 #endif
@@ -3957,7 +3957,7 @@ static void init_all(Tcl_Interp *interp){
     TestSession_Init(interp);
 #endif
     Fts5tcl_Init(interp);
-    SqliteOta_Init(interp);
+    SqliteRbu_Init(interp);
 
 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
     Sqlitetestfts3_Init(interp);
diff --git a/src/test1.c b/src/test1.c
index 3885b71ca..15fd8c7f7 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -274,6 +274,9 @@ static int clang_sanitize_address(
   res = 1;
 # endif
 #endif
+#ifdef __SANITIZE_ADDRESS__
+  res = 1;
+#endif
   if( res==0 && getenv("OMIT_MISUSE")!=0 ) res = 1;
   Tcl_SetObjResult(interp, Tcl_NewIntObj(res));
   return TCL_OK;
diff --git a/src/test_config.c b/src/test_config.c
index eb30e2d25..1335ced67 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -442,10 +442,10 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "or_opt", "1", TCL_GLOBAL_ONLY);
 #endif
 
-#ifdef SQLITE_ENABLE_OTA
-  Tcl_SetVar2(interp, "sqlite_options", "ota", "1", TCL_GLOBAL_ONLY);
+#ifdef SQLITE_ENABLE_RBU
+  Tcl_SetVar2(interp, "sqlite_options", "rbu", "1", TCL_GLOBAL_ONLY);
 #else
-  Tcl_SetVar2(interp, "sqlite_options", "ota", "0", TCL_GLOBAL_ONLY);
+  Tcl_SetVar2(interp, "sqlite_options", "rbu", "0", TCL_GLOBAL_ONLY);
 #endif
 
 #ifdef SQLITE_OMIT_PAGER_PRAGMAS
diff --git a/src/wal.c b/src/wal.c
index ce98149e3..f7e259400 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -648,9 +648,9 @@ static void walIndexWriteHdr(Wal *pWal){
   pWal->hdr.isInit = 1;
   pWal->hdr.iVersion = WALINDEX_MAX_VERSION;
   walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum);
-  memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr));
+  memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr));
   walShmBarrier(pWal);
-  memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr));
+  memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr));
 }
 
 /*
@@ -952,13 +952,13 @@ static void walCleanupHash(Wal *pWal){
   ** via the hash table even after the cleanup.
   */
   if( iLimit ){
-    int i;           /* Loop counter */
+    int j;           /* Loop counter */
     int iKey;        /* Hash key */
-    for(i=1; i<=iLimit; i++){
-      for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){
-        if( aHash[iKey]==i ) break;
+    for(j=1; j<=iLimit; j++){
+      for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){
+        if( aHash[iKey]==j ) break;
       }
-      assert( aHash[iKey]==i );
+      assert( aHash[iKey]==j );
     }
   }
 #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */
@@ -1460,7 +1460,7 @@ static void walMergesort(
   int nMerge = 0;                 /* Number of elements in list aMerge */
   ht_slot *aMerge = 0;            /* List to be merged */
   int iList;                      /* Index into input list */
-  int iSub = 0;                   /* Index into aSub array */
+  u32 iSub = 0;                   /* Index into aSub array */
   struct Sublist aSub[13];        /* Array of sub-lists */
 
   memset(aSub, 0, sizeof(aSub));
@@ -1471,7 +1471,9 @@ static void walMergesort(
     nMerge = 1;
     aMerge = &aList[iList];
     for(iSub=0; iList & (1<<iSub); iSub++){
-      struct Sublist *p = &aSub[iSub];
+      struct Sublist *p;
+      assert( iSub<ArraySize(aSub) );
+      p = &aSub[iSub];
       assert( p->aList && p->nList<=(1<<iSub) );
       assert( p->aList==&aList[iList&~((2<<iSub)-1)] );
       walMerge(aContent, p->aList, p->nList, &aMerge, &nMerge, aBuffer);
@@ -1482,7 +1484,9 @@ static void walMergesort(
 
   for(iSub++; iSub<ArraySize(aSub); iSub++){
     if( nList & (1<<iSub) ){
-      struct Sublist *p = &aSub[iSub];
+      struct Sublist *p;
+      assert( iSub<ArraySize(aSub) );
+      p = &aSub[iSub];
       assert( p->nList<=(1<<iSub) );
       assert( p->aList==&aList[nList&~((2<<iSub)-1)] );
       walMerge(aContent, p->aList, p->nList, &aMerge, &nMerge, aBuffer);
diff --git a/src/wherecode.c b/src/wherecode.c
index e1f0f8661..9747f7f37 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1296,7 +1296,11 @@ Bitmask sqlite3WhereCodeOneLoopStart(
               r = sqlite3GetTempRange(pParse, nPk);
               for(iPk=0; iPk<nPk; iPk++){
                 int iCol = pPk->aiColumn[iPk];
-                sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur, r+iPk, 0);
+                int rx;
+                rx = sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur,r+iPk,0);
+                if( rx!=r+iPk ){
+                  sqlite3VdbeAddOp2(v, OP_SCopy, rx, r+iPk);
+                }
               }
 
               /* Check if the temp table already contains this key. If so,