Merge current trunk into this branch.

FossilOrigin-Name: ed47d7f9a44b2af8ab7dd956495e71ea9a159cec438d1909f7022254a779e068

28 files changed, 2291 insertions, 520 deletions

diff --git a/ext/consio/console_io.c b/ext/consio/console_io.c
index 3fa613ba9..75324a34f 100755
--- a/ext/consio/console_io.c
+++ b/ext/consio/console_io.c
@@ -595,6 +595,93 @@ oPutbUtf8(const char *cBuf, int nAccept){
 # endif
 }
 
+/*
+** Flush the given output stream. Return non-zero for success, else 0.
+*/
+#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE)
+SQLITE_INTERNAL_LINKAGE int
+fFlushBuffer(FILE *pfOut){
+# if CIO_WIN_WC_XLATE && !defined(SHELL_OMIT_FIO_DUPE)
+  return FlushFileBuffers(handleOfFile(pfOut))? 1 : 0;
+# else
+  return fflush(pfOut);
+# endif
+}
+#endif
+
+#if CIO_WIN_WC_XLATE \
+   && !defined(SHELL_OMIT_FIO_DUPE) \
+   && defined(SQLITE_USE_ONLY_WIN32)
+static struct FileAltIds {
+  int fd;
+  HANDLE fh;
+} altIdsOfFile(FILE *pf){
+  struct FileAltIds rv = { _fileno(pf) };
+  union { intptr_t osfh; HANDLE fh; } fid = {
+    (rv.fd>=0)? _get_osfhandle(rv.fd) : (intptr_t)INVALID_HANDLE_VALUE
+  };
+  rv.fh = fid.fh;
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE size_t
+cfWrite(const void *buf, size_t osz, size_t ocnt, FILE *pf){
+  size_t rv = 0;
+  struct FileAltIds fai = altIdsOfFile(pf);
+  int fmode = _setmode(fai.fd, _O_BINARY);
+  _setmode(fai.fd, fmode);
+  while( rv < ocnt ){
+    size_t nbo = osz;
+    while( nbo > 0 ){
+      DWORD dwno = (nbo>(1L<<24))? 1L<<24 : (DWORD)nbo;
+      BOOL wrc = TRUE;
+      BOOL genCR = (fmode & _O_TEXT)!=0;
+      if( genCR ){
+        const char *pnl = (const char*)memchr(buf, '\n', nbo);
+        if( pnl ) nbo = pnl - (const char*)buf;
+        else genCR = 0;
+      }
+      if( dwno>0 ) wrc = WriteFile(fai.fh, buf, dwno, 0,0);
+      if( genCR && wrc ){
+        wrc = WriteFile(fai.fh, "\r\n", 2, 0,0);
+        ++dwno; /* Skip over the LF */
+      }
+      if( !wrc ) return rv;
+      buf = (const char*)buf + dwno;
+      nbo += dwno;
+    }
+    ++rv;
+  }
+  return rv;
+}
+
+/* An fgets() equivalent, using Win32 file API for actual input.
+** Input ends when given buffer is filled or a newline is read.
+** If the FILE object is in text mode, swallows 0x0D. (ASCII CR)
+*/
+SQLITE_INTERNAL_LINKAGE char *
+cfGets(char *cBuf, int n, FILE *pf){
+  int nci = 0;
+  struct FileAltIds fai = altIdsOfFile(pf);
+  int fmode = _setmode(fai.fd, _O_BINARY);
+  BOOL eatCR = (fmode & _O_TEXT)!=0;
+  _setmode(fai.fd, fmode);
+  while( nci < n-1 ){
+    DWORD nr;
+    if( !ReadFile(fai.fh, cBuf+nci, 1, &nr, 0) || nr==0 ) break;
+    if( nr>0 && (!eatCR || cBuf[nci]!='\r') ){
+      nci += nr;
+      if( cBuf[nci-nr]=='\n' ) break;
+    }
+  }
+  if( nci < n ) cBuf[nci] = 0;
+  return (nci>0)? cBuf : 0;
+}
+# else
+#  define cfWrite(b,os,no,f) fwrite(b,os,no,f)
+#  define cfGets(b,n,f) fgets(b,n,f)
+# endif
+
 # ifdef CONSIO_EPUTB
 SQLITE_INTERNAL_LINKAGE int
 ePutbUtf8(const char *cBuf, int nAccept){
@@ -606,7 +693,7 @@ ePutbUtf8(const char *cBuf, int nAccept){
     return conZstrEmit(ppst, cBuf, nAccept);
   }else {
 #  endif
-    return (int)fwrite(cBuf, 1, nAccept, pfErr);
+    return (int)cfWrite(cBuf, 1, nAccept, pfErr);
 #  if CIO_WIN_WC_XLATE
   }
 #  endif
@@ -672,7 +759,7 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
 #  endif
   }else{
 # endif
-    return fgets(cBuf, ncMax, pfIn);
+    return cfGets(cBuf, ncMax, pfIn);
 # if CIO_WIN_WC_XLATE
   }
 # endif
diff --git a/ext/consio/console_io.h b/ext/consio/console_io.h
index 26fd7dd94..1affa15ba 100644
--- a/ext/consio/console_io.h
+++ b/ext/consio/console_io.h
@@ -177,11 +177,18 @@ ePutbUtf8(const char *cBuf, int nAccept);
 #endif
 
 /*
+** Flush the given output stream. Return non-zero for success, else 0.
+*/
+#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE)
+SQLITE_INTERNAL_LINKAGE int
+fFlushBuffer(FILE *pfOut);
+#endif
+
+/*
 ** Collect input like fgets(...) with special provisions for input
-** from the console on platforms that require same. Defers to the
-** C library fgets() when input is not from the console. Newline
-** translation may be done as set by set{Binary,Text}Mode(). As a
-** convenience, pfIn==NULL is treated as stdin.
+** from the console on such platforms as require same. Newline
+** translation may be done as set by set{Binary,Text}Mode().
+** As a convenience, pfIn==NULL is treated as stdin.
 */
 SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
 /* Like fGetsUtf8 except stream is always the designated input. */
diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h
index 67adb40ac..ea69f8f02 100644
--- a/ext/fts5/fts5.h
+++ b/ext/fts5/fts5.h
@@ -493,18 +493,19 @@ struct Fts5ExtensionApi {
 **
 ** FTS5_TOKENIZER
 **
-** There is also an fts5_tokenizer object. This is an older version of
-** fts5_tokenizer_v2. It is similar except that:
+** There is also an fts5_tokenizer object. This is an older, deprecated,
+** version of fts5_tokenizer_v2. It is similar except that:
 **
 **  <ul>
 **    <li> There is no "iVersion" field, and
 **    <li> The xTokenize() method does not take a locale argument.
 **  </ul>
 **
-** fts5_tokenizer tokenizers should be registered with the xCreateTokenizer()
-** function, instead of xCreateTokenizer_v2(). Tokenizers implementations 
-** registered using either API may be retrieved using both xFindTokenizer()
-** and xFindTokenizer_v2().
+** Legacy fts5_tokenizer tokenizers must be registered using the
+** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().
+**
+** Tokenizer implementations registered using either API may be retrieved
+** using both xFindTokenizer() and xFindTokenizer_v2().
 **
 ** SYNONYM SUPPORT
 **
diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 7e4111957..b15521f16 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -636,16 +636,13 @@ Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
 
 int sqlite3Fts5FlushToDisk(Fts5Table*);
 
-int sqlite3Fts5ExtractText(
-  Fts5Config *pConfig,
-  sqlite3_value *pVal,            /* Value to extract text from */
-  int bContent,                   /* Loaded from content table */
-  int *pbResetTokenizer,          /* OUT: True if ClearLocale() required */
-  const char **ppText,            /* OUT: Pointer to text buffer */
-  int *pnText                     /* OUT: Size of (*ppText) in bytes */
-);
-
 void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
+void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc);
+
+int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);
+int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal, 
+    const char **ppText, int *pnText, const char **ppLoc, int *pnLoc
+);
 
 /*
 ** End of interface to code in fts5.c.
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index 3cb1bd3be..1fade5f7a 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -516,6 +516,15 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){
       }
     }
   }
+  if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){
+    for(i=0; i<p->nCol; i++){
+      if( p->abUnindexed[i]==0 ){
+        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i);
+      }else{
+        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
+      }
+    }
+  }
 
   assert( p->zContentExprlist==0 );
   p->zContentExprlist = (char*)buf.p;
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 1f0a68d3e..4363305a5 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -2185,7 +2185,7 @@ static void fts5SegIterNext_None(
 
   if( iOff<pIter->iEndofDoclist ){
     /* Next entry is on the current page */
-    i64 iDelta;
+    u64 iDelta;
     iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta);
     pIter->iLeafOffset = iOff;
     pIter->iRowid += iDelta;
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 4addc0729..db413b572 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -83,9 +83,18 @@ struct Fts5Global {
   Fts5TokenizerModule *pTok;      /* First in list of all tokenizer modules */
   Fts5TokenizerModule *pDfltTok;  /* Default tokenizer module */
   Fts5Cursor *pCsr;               /* First in list of all open cursors */
+  u32 aLocaleHdr[4];
 };
 
 /*
+** Size of header on fts5_locale() values. And macro to access a buffer
+** containing a copy of the header from an Fts5Config pointer.
+*/
+#define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
+#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr))
+
+
+/*
 ** Each auxiliary function registered with the FTS5 module is represented
 ** by an object of the following type. All such objects are stored as part
 ** of the Fts5Global.pAux list.
@@ -247,12 +256,6 @@ struct Fts5Cursor {
 #define BitFlagAllTest(x,y) (((x) & (y))==(y))
 #define BitFlagTest(x,y)    (((x) & (y))!=0)
 
-/*
-** The subtype value and header bytes used by fts5_locale().
-*/
-#define FTS5_LOCALE_SUBTYPE ((unsigned int)'L')
-#define FTS5_LOCALE_HEADER  "\x00\xE0\xB2\xEB"
-
 
 /*
 ** Macros to Set(), Clear() and Test() cursor flags.
@@ -427,8 +430,7 @@ static int fts5InitVtab(
 
   /* Load the initial configuration */
   if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
-    sqlite3Fts5IndexRollback(pTab->p.pIndex);
+    rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1);
   }
 
   if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
@@ -1257,7 +1259,7 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
 ** valid until after the final call to sqlite3Fts5Tokenize() that will use
 ** the locale.
 */
-static void fts5SetLocale(
+static void sqlite3Fts5SetLocale(
   Fts5Config *pConfig, 
   const char *zLocale, 
   int nLocale
@@ -1268,127 +1270,74 @@ static void fts5SetLocale(
 }
 
 /*
-** Clear any locale configured by an earlier call to fts5SetLocale() or
-** sqlite3Fts5ExtractText().
+** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale().
 */
 void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
-  fts5SetLocale(pConfig, 0, 0);
+  sqlite3Fts5SetLocale(pConfig, 0, 0);
 }
 
 /*
-** This function is used to extract utf-8 text from an sqlite3_value. This
-** is usually done in order to tokenize it. For example, when:
-**
-**     * a value is written to an fts5 table,
-**     * a value is deleted from an FTS5_CONTENT_NORMAL table,
-**     * a value containing a query expression is passed to xFilter()
-**
-** and so on.
-**
-** This function handles 2 cases:
-**
-**   1) Ordinary values. The text can be extracted from these using
-**      sqlite3_value_text().
-**
-**   2) Combination text/locale blobs created by fts5_locale(). There
-**      are several cases for these:
-**
-**        * Blobs tagged with FTS5_LOCALE_SUBTYPE.
-**        * Blobs read from the content table of a locale=1 external-content
-**          table, and 
-**        * Blobs read from the content table of a locale=1 regular 
-**          content table.
-**
-**      The first two cases above should have the 4 byte FTS5_LOCALE_HEADER
-**      header. It is an error if a blob with the subtype or a blob read
-**      from the content table of an external content table does not have
-**      the required header. A blob read from the content table of a regular
-**      locale=1 table does not have the header. This is to save space.
-**
-** If successful, SQLITE_OK is returned and output parameters (*ppText)
-** and (*pnText) are set to point to a buffer containing the extracted utf-8 
-** text and its length in bytes, respectively. The buffer is not 
-** nul-terminated. It has the same lifetime as the sqlite3_value object
-** from which it is extracted.
-**
-** Parameter bContent must be true if the value was read from an indexed
-** column (i.e. not UNINDEXED) of the on disk content. 
-**
-** If pbResetTokenizer is not NULL and if case (2) is used, then 
-** fts5SetLocale() is called to ensure subsequent sqlite3Fts5Tokenize() calls
-** use the locale. In this case (*pbResetTokenizer) is set to true before
-** returning, to indicate that the caller must call sqlite3Fts5ClearLocale() 
-** to clear the locale after tokenizing the text.
+** Return true if the value passed as the only argument is an 
+** fts5_locale() value.  
 */
-int sqlite3Fts5ExtractText(
-  Fts5Config *pConfig,
-  sqlite3_value *pVal,            /* Value to extract text from */
-  int bContent,                   /* True if indexed table content */
-  int *pbResetTokenizer,          /* OUT: True if xSetLocale(NULL) required */
-  const char **ppText,            /* OUT: Pointer to text buffer */
-  int *pnText                     /* OUT: Size of (*ppText) in bytes */
-){
-  const char *pText = 0;
-  int nText = 0;
-  int rc = SQLITE_OK;
-  int bDecodeBlob = 0;
-
-  assert( pbResetTokenizer==0 || *pbResetTokenizer==0 );
-  assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE );
-  assert( bContent==0 || sqlite3_value_subtype(pVal)==0 );
-
+int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){
+  int ret = 0;
   if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
-    if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE 
-     || (bContent && pConfig->bLocale)
+    /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case.
+    ** If the blob was created using zeroblob(), then sqlite3_value_blob()
+    ** may call malloc(). If this malloc() fails, then the values returned
+    ** by both value_blob() and value_bytes() will be 0. If value_bytes() were 
+    ** called first, then the NULL pointer returned by value_blob() might
+    ** be dereferenced.  */
+    const u8 *pBlob = sqlite3_value_blob(pVal);
+    int nBlob = sqlite3_value_bytes(pVal); 
+    if( nBlob>FTS5_LOCALE_HDR_SIZE
+     && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE)
     ){
-      bDecodeBlob = 1;
+      ret = 1;
     }
   }
+  return ret;
+}
 
-  if( bDecodeBlob ){
-    const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
-    const u8 *pBlob = sqlite3_value_blob(pVal);
-    int nBlob = sqlite3_value_bytes(pVal);
-
-    /* Unless this blob was read from the %_content table of an 
-    ** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale() 
-    ** header. Check for this. If it is not found, return an error.  */
-    if( (!bContent || pConfig->eContent!=FTS5_CONTENT_NORMAL) ){
-      if( nBlob<SZHDR || memcmp(FTS5_LOCALE_HEADER, pBlob, SZHDR) ){
-        rc = SQLITE_ERROR;
-      }else{
-        pBlob += 4;
-        nBlob -= 4;
-      }
-    }
-
-    if( rc==SQLITE_OK ){
-      int nLocale = 0;
+/*
+** Value pVal is guaranteed to be an fts5_locale() value, according to
+** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale
+** from the value and returns them separately.
+**
+** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set
+** to point to buffers containing the text and locale, as utf-8,
+** respectively. In this case output parameters (*pnText) and (*pnLoc) are
+** set to the sizes in bytes of these two buffers.
+**
+** Or, if an error occurs, then an SQLite error code is returned. The final
+** value of the four output parameters is undefined in this case.
+*/
+int sqlite3Fts5DecodeLocaleValue(
+  sqlite3_value *pVal, 
+  const char **ppText,
+  int *pnText, 
+  const char **ppLoc, 
+  int *pnLoc
+){
+  const char *p = sqlite3_value_blob(pVal);
+  int n = sqlite3_value_bytes(pVal);
+  int nLoc = 0;
 
-      for(nLocale=0; nLocale<nBlob; nLocale++){
-        if( pBlob[nLocale]==0x00 ) break;
-      }
-      if( nLocale==nBlob || nLocale==0 ){
-        rc = SQLITE_ERROR;
-      }else{
-        pText = (const char*)&pBlob[nLocale+1];
-        nText = nBlob-nLocale-1;
+  assert( sqlite3_value_type(pVal)==SQLITE_BLOB );
+  assert( n>FTS5_LOCALE_HDR_SIZE );
 
-        if( pbResetTokenizer ){
-          fts5SetLocale(pConfig, (const char*)pBlob, nLocale);
-          *pbResetTokenizer = 1;
-        }
-      }
+  for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){
+    if( nLoc==(n-1) ){
+      return SQLITE_MISMATCH;
     }
-
-  }else{
-    pText = (const char*)sqlite3_value_text(pVal);
-    nText = sqlite3_value_bytes(pVal);
   }
+  *ppLoc = &p[FTS5_LOCALE_HDR_SIZE];
+  *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE;
 
-  *ppText = pText;
-  *pnText = nText;
-  return rc;
+  *ppText = &p[nLoc+1];
+  *pnText = n - nLoc - 1;
+  return SQLITE_OK;
 }
 
 /*
@@ -1397,8 +1346,8 @@ int sqlite3Fts5ExtractText(
 ** the text of the expression, and sets output variable (*pzText) to 
 ** point to a nul-terminated buffer containing the expression.
 **
-** If pVal was an fts5_locale() value, then fts5SetLocale() is called to
-** set the tokenizer to use the specified locale.
+** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called 
+** to set the tokenizer to use the specified locale.
 **
 ** If output variable (*pbFreeAndReset) is set to true, then the caller
 ** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer 
@@ -1410,24 +1359,22 @@ static int fts5ExtractExprText(
   char **pzText,                  /* OUT: nul-terminated buffer of text */
   int *pbFreeAndReset             /* OUT: Free (*pzText) and clear locale */
 ){
-  const char *zText = 0;
-  int nText = 0;
   int rc = SQLITE_OK;
-  int bReset = 0;
 
-  *pbFreeAndReset = 0;
-  rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, &bReset, &zText, &nText);
-  if( rc==SQLITE_OK ){
-    if( bReset ){
-      *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText);
-      if( rc!=SQLITE_OK ){
-        sqlite3Fts5ClearLocale(pConfig);
-      }else{
-        *pbFreeAndReset = 1;
-      }
-    }else{
-      *pzText = (char*)zText;
+  if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
+    const char *pText = 0;
+    int nText = 0;
+    const char *pLoc = 0;
+    int nLoc = 0;
+    rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+    *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText);
+    if( rc==SQLITE_OK ){
+      sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
     }
+    *pbFreeAndReset = 1;
+  }else{
+    *pzText = (char*)sqlite3_value_text(pVal);
+    *pbFreeAndReset = 0;
   }
 
   return rc;
@@ -1967,19 +1914,14 @@ static int fts5UpdateMethod(
     else{
       int eType1 = sqlite3_value_numeric_type(apVal[1]);
 
-      /* Ensure that no fts5_locale() values are written to locale=0 tables.
-      ** And that no blobs except fts5_locale() blobs are written to indexed
-      ** (i.e. not UNINDEXED) columns of locale=1 tables. */
-      int ii;
-      for(ii=0; ii<pConfig->nCol; ii++){
-        if( sqlite3_value_type(apVal[ii+2])==SQLITE_BLOB ){
-          int bSub = (sqlite3_value_subtype(apVal[ii+2])==FTS5_LOCALE_SUBTYPE);
-          if( (pConfig->bLocale && !bSub && pConfig->abUnindexed[ii]==0) 
-           || (pConfig->bLocale==0 && bSub)
-          ){
-            if( pConfig->bLocale==0 ){
-              fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
-            }
+      /* It is an error to write an fts5_locale() value to a table without
+      ** the locale=1 option. */
+      if( pConfig->bLocale==0 ){
+        int ii;
+        for(ii=0; ii<pConfig->nCol; ii++){
+          sqlite3_value *pVal = apVal[ii+2];
+          if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
+            fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
             rc = SQLITE_MISMATCH;
             goto update_out;
           }
@@ -2068,9 +2010,11 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
 ** Implementation of xBegin() method. 
 */
 static int fts5BeginMethod(sqlite3_vtab *pVtab){
-  fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0);
-  fts5NewTransaction((Fts5FullTable*)pVtab);
-  return SQLITE_OK;
+  int rc = fts5NewTransaction((Fts5FullTable*)pVtab);
+  if( rc==SQLITE_OK ){
+    fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0);
+  }
+  return rc;
 }
 
 /*
@@ -2138,11 +2082,11 @@ static int fts5ApiTokenize_v2(
   Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
   int rc = SQLITE_OK;
 
-  fts5SetLocale(pTab->pConfig, pLoc, nLoc);
+  sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc);
   rc = sqlite3Fts5Tokenize(pTab->pConfig, 
       FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken
   );
-  fts5SetLocale(pTab->pConfig, 0, 0);
+  sqlite3Fts5SetLocale(pTab->pConfig, 0, 0);
 
   return rc;
 }
@@ -2170,6 +2114,49 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
   return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
 }
 
+/*
+** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This
+** function extracts the text value of column iCol of the current row.
+** Additionally, if there is an associated locale, it invokes
+** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller
+** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point
+** after this function returns.
+**
+** If successful, (*ppText) is set to point to a buffer containing the text
+** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that
+** buffer in bytes. It is not guaranteed to be nul-terminated. If an error
+** occurs, an SQLite error code is returned. The final values of the two
+** output parameters are undefined in this case.
+*/
+static int fts5TextFromStmt(
+  Fts5Config *pConfig,
+  sqlite3_stmt *pStmt,
+  int iCol,
+  const char **ppText,
+  int *pnText
+){
+  sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1);
+  const char *pLoc = 0;
+  int nLoc = 0;
+  int rc = SQLITE_OK;
+
+  if( pConfig->bLocale 
+   && pConfig->eContent==FTS5_CONTENT_EXTERNAL
+   && sqlite3Fts5IsLocaleValue(pConfig, pVal) 
+  ){
+    rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc);
+  }else{
+    *ppText = (const char*)sqlite3_value_text(pVal);
+    *pnText = sqlite3_value_bytes(pVal);
+    if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){
+      pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol);
+      nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol);
+    }
+  }
+  sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
+  return rc;
+}
+
 static int fts5ApiColumnText(
   Fts5Context *pCtx, 
   int iCol, 
@@ -2189,10 +2176,8 @@ static int fts5ApiColumnText(
   }else{
     rc = fts5SeekCursor(pCsr, 0);
     if( rc==SQLITE_OK ){
-      Fts5Config *pConfig = pTab->pConfig;
-      int bContent = (pConfig->abUnindexed[iCol]==0);
-      sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
-      sqlite3Fts5ExtractText(pConfig, pVal, bContent, 0, pz, pn);
+      rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn);
+      sqlite3Fts5ClearLocale(pTab->pConfig);
     }
   }
   return rc;
@@ -2234,17 +2219,15 @@ static int fts5CsrPoslist(
         rc = fts5SeekCursor(pCsr, 0);
       }
       for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
-        sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1);
         const char *z = 0;
         int n = 0; 
-        int bReset = 0;
-        rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n);
+        rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
         if( rc==SQLITE_OK ){
           rc = sqlite3Fts5ExprPopulatePoslists(
               pConfig, pCsr->pExpr, aPopulator, i, z, n
           );
         }
-        if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+        sqlite3Fts5ClearLocale(pConfig);
       }
       sqlite3_free(aPopulator);
 
@@ -2430,17 +2413,14 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
         if( pConfig->abUnindexed[i]==0 ){
           const char *z = 0; 
           int n = 0;
-          int bReset = 0;
-          sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1);
-
           pCsr->aColumnSize[i] = 0;
-          rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n);
+          rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
           if( rc==SQLITE_OK ){
             rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, 
                 z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb
             );
-            if( bReset ) sqlite3Fts5ClearLocale(pConfig);
           }
+          sqlite3Fts5ClearLocale(pConfig);
         }
       }
     }
@@ -2712,37 +2692,14 @@ static int fts5ApiColumnLocale(
   ){
     rc = fts5SeekCursor(pCsr, 0);
     if( rc==SQLITE_OK ){
-      /* Load the value into pVal. pVal is a locale/text pair iff:
-      **
-      **   1) It is an SQLITE_BLOB, and
-      **   2) Either the subtype is FTS5_LOCALE_SUBTYPE, or else the
-      **      value was loaded from an FTS5_CONTENT_NORMAL table, and
-      **   3) It does not begin with an 0x00 byte.
-      */ 
-      sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
-      if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
-        const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
-        int nBlob = sqlite3_value_bytes(pVal);
-        if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
-          const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
-          if( nBlob<SZHDR || memcmp(FTS5_LOCALE_HEADER, pBlob, SZHDR) ){
-            rc = SQLITE_ERROR;
-          }
-          pBlob += 4;
-          nBlob -= 4;
-        }
-        if( rc==SQLITE_OK ){
-          int nLocale = 0;
-          for(nLocale=0; nLocale<nBlob && pBlob[nLocale]!=0x00; nLocale++);
-          if( nLocale==nBlob || nLocale==0 ){
-            rc = SQLITE_ERROR;
-          }else{
-            /* A locale/text pair */
-            *pzLocale = (const char*)pBlob;
-            *pnLocale = nLocale;
-          }
-        }
+      const char *zDummy = 0;
+      int nDummy = 0;
+      rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy);
+      if( rc==SQLITE_OK ){
+        *pzLocale = pConfig->t.pLocale;
+        *pnLocale = pConfig->t.nLocale;
       }
+      sqlite3Fts5ClearLocale(pConfig);
     }
   }
 
@@ -2963,57 +2920,6 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
   return rc;
 }
 
-/*
-** Value pVal was read from column iCol of the FTS5 table. This function
-** returns it to the owner of pCtx via a call to an sqlite3_result_xxx()
-** function. This function deals with the same cases as 
-** sqlite3Fts5ExtractText():
-**
-**   1) Ordinary values. These can be returned using sqlite3_result_value().
-**
-**   2) Blobs from fts5_locale(). The text is extracted from these and 
-**      returned via sqlite3_result_text(). The locale is discarded.
-*/
-static void fts5ExtractValueFromColumn(
-  sqlite3_context *pCtx,
-  Fts5Config *pConfig, 
-  int iCol,
-  sqlite3_value *pVal
-){
-  assert( pConfig->eContent!=FTS5_CONTENT_NONE );
-
-  if( pConfig->bLocale 
-   && sqlite3_value_type(pVal)==SQLITE_BLOB 
-   && pConfig->abUnindexed[iCol]==0
-  ){
-    const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
-    const u8 *pBlob = sqlite3_value_blob(pVal);
-    int nBlob = sqlite3_value_bytes(pVal);
-    int ii;
-
-    if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
-      if( nBlob<SZHDR || memcmp(pBlob, FTS5_LOCALE_HEADER, SZHDR) ){
-        sqlite3_result_error_code(pCtx, SQLITE_ERROR);
-        return;
-      }else{
-        pBlob += 4;
-        nBlob -= 4;
-      }
-    }
-
-    for(ii=0; ii<nBlob && pBlob[ii]; ii++);
-    if( ii==0 || ii==nBlob ){
-      sqlite3_result_error_code(pCtx, SQLITE_ERROR);
-    }else{
-      const char *pText = (const char*)&pBlob[ii+1];
-      sqlite3_result_text(pCtx, pText, nBlob-ii-1, SQLITE_TRANSIENT);
-    }
-    return;
-  }
-
-  sqlite3_result_value(pCtx, pVal);
-}
-
 /* 
 ** This is the xColumn method, called by SQLite to request a value from
 ** the row that the supplied cursor currently points to.
@@ -3070,8 +2976,22 @@ static int fts5ColumnMethod(
       rc = fts5SeekCursor(pCsr, 1);
       if( rc==SQLITE_OK ){
         sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
-        fts5ExtractValueFromColumn(pCtx, pConfig, iCol, pVal);
+        if( pConfig->bLocale 
+         && pConfig->eContent==FTS5_CONTENT_EXTERNAL 
+         && sqlite3Fts5IsLocaleValue(pConfig, pVal)
+        ){
+          const char *z = 0;
+          int n = 0;
+          rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n);
+          if( rc==SQLITE_OK ){
+            sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT);
+          }
+          sqlite3Fts5ClearLocale(pConfig);
+        }else{
+          sqlite3_result_value(pCtx, pVal);
+        }
       }
+
       pConfig->pzErrmsg = 0;
     }
   }
@@ -3267,7 +3187,9 @@ static int fts5NewTokenizerModule(
 */
 typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer;
 struct Fts5VtoVTokenizer {
-  Fts5TokenizerModule *pMod;
+  int bV2Native;                  /* True if v2 native tokenizer */
+  fts5_tokenizer x1;              /* Tokenizer functions */
+  fts5_tokenizer_v2 x2;           /* V2 tokenizer functions */
   Fts5Tokenizer *pReal;
 };
 
@@ -3287,7 +3209,9 @@ static int fts5VtoVCreate(
 
   pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
   if( rc==SQLITE_OK ){
-    pNew->pMod = pMod;
+    pNew->x1 = pMod->x1;
+    pNew->x2 = pMod->x2;
+    pNew->bV2Native = pMod->bV2Native;
     if( pMod->bV2Native ){
       rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
     }else{
@@ -3309,11 +3233,10 @@ static int fts5VtoVCreate(
 static void fts5VtoVDelete(Fts5Tokenizer *pTok){
   Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
   if( p ){
-    Fts5TokenizerModule *pMod = p->pMod;
-    if( pMod->bV2Native ){
-      pMod->x2.xDelete(p->pReal);
+    if( p->bV2Native ){
+      p->x2.xDelete(p->pReal);
     }else{
-      pMod->x1.xDelete(p->pReal);
+      p->x1.xDelete(p->pReal);
     }
     sqlite3_free(p);
   }
@@ -3331,9 +3254,8 @@ static int fts5V1toV2Tokenize(
   int (*xToken)(void*, int, const char*, int, int, int)
 ){
   Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
-  Fts5TokenizerModule *pMod = p->pMod;
-  assert( pMod->bV2Native );
-  return pMod->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
+  assert( p->bV2Native );
+  return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
 }
 
 /*
@@ -3348,10 +3270,9 @@ static int fts5V2toV1Tokenize(
   int (*xToken)(void*, int, const char*, int, int, int)
 ){
   Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
-  Fts5TokenizerModule *pMod = p->pMod;
-  assert( pMod->bV2Native==0 );
+  assert( p->bV2Native==0 );
   UNUSED_PARAM2(pLocale,nLocale);
-  return pMod->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
+  return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
 }
 
 /*
@@ -3632,13 +3553,12 @@ static void fts5LocaleFunc(
   if( zLocale==0 || zLocale[0]=='\0' ){
     sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT);
   }else{
+    Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx);
     u8 *pBlob = 0;
     u8 *pCsr = 0;
     int nBlob = 0;
-    const int nHdr = 4;
-    assert( sizeof(FTS5_LOCALE_HEADER)==nHdr+1 );
 
-    nBlob = nHdr + nLocale + 1 + nText;
+    nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText;
     pBlob = (u8*)sqlite3_malloc(nBlob);
     if( pBlob==0 ){
       sqlite3_result_error_nomem(pCtx);
@@ -3646,8 +3566,8 @@ static void fts5LocaleFunc(
     }
 
     pCsr = pBlob;
-    memcpy(pCsr, FTS5_LOCALE_HEADER, nHdr);
-    pCsr += nHdr;
+    memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE);
+    pCsr += FTS5_LOCALE_HDR_SIZE;
     memcpy(pCsr, zLocale, nLocale);
     pCsr += nLocale;
     (*pCsr++) = 0x00;
@@ -3655,7 +3575,6 @@ static void fts5LocaleFunc(
     assert( &pCsr[nText]==&pBlob[nBlob] );
 
     sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free);
-    sqlite3_result_subtype(pCtx, FTS5_LOCALE_SUBTYPE);
   }
 }
 
@@ -3757,6 +3676,16 @@ static int fts5Init(sqlite3 *db){
     pGlobal->api.xFindTokenizer = fts5FindTokenizer;
     pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2;
     pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2;
+
+    /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector.
+    ** The constants below were generated randomly.  */
+    sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr);
+    pGlobal->aLocaleHdr[0] ^= 0xF924976D;
+    pGlobal->aLocaleHdr[1] ^= 0x16596E13;
+    pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA;
+    pGlobal->aLocaleHdr[3] ^= 0x9B03A67F;
+    assert( sizeof(pGlobal->aLocaleHdr)==16 );
+
     rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
     if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
     if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index cf25eb361..e8649c703 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -74,6 +74,30 @@ struct Fts5Storage {
 #define FTS5_STMT_SCAN           11
 
 /*
+** Return a pointer to a buffer obtained from sqlite3_malloc() that contains
+** nBind comma-separated question marks. e.g. if nBind is passed 5, this 
+** function returns "?,?,?,?,?".
+**
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op and
+** NULL is returned immediately. Or, if the attempt to malloc a buffer
+** fails, then *pRc is set to SQLITE_NOMEM and NULL is returned. Otherwise,
+** if it is SQLITE_OK when this function is called and the malloc() succeeds,
+** *pRc is left unchanged.
+*/
+static char *fts5BindingsList(int *pRc, int nBind){
+  char *zBind = sqlite3Fts5MallocZero(pRc, 1 + nBind*2);
+  if( zBind ){
+    int ii;
+    for(ii=0; ii<nBind; ii++){
+      zBind[ii*2] = '?';
+      zBind[ii*2 + 1] = ',';
+    }
+    zBind[ii*2-1] = '\0';
+  }
+  return zBind;
+}
+
+/*
 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
 ** Fts5Storage.pInsertDocsize - if they have not already been prepared.
 ** Return SQLITE_OK if successful, or an SQLite error code if an error
@@ -141,19 +165,20 @@ static int fts5StorageGetStmt(
         );
         break;
 
-      case FTS5_STMT_INSERT_CONTENT: 
-      case FTS5_STMT_REPLACE_CONTENT: {
-        int nCol = pC->nCol + 1;
+      case FTS5_STMT_INSERT_CONTENT: {
+        int nCol = 0;
         char *zBind;
         int i;
 
-        zBind = sqlite3_malloc64(1 + nCol*2);
-        if( zBind ){
-          for(i=0; i<nCol; i++){
-            zBind[i*2] = '?';
-            zBind[i*2 + 1] = ',';
+        nCol = 1 + pC->nCol;
+        if( pC->bLocale ){
+          for(i=0; i<pC->nCol; i++){
+            if( pC->abUnindexed[i]==0 ) nCol++;
           }
-          zBind[i*2-1] = '\0';
+        }
+
+        zBind = fts5BindingsList(&rc, nCol);
+        if( zBind ){
           zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
           sqlite3_free(zBind);
         }
@@ -344,7 +369,7 @@ int sqlite3Fts5StorageOpen(
   if( bCreate ){
     if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
       int nDefn = 32 + pConfig->nCol*10;
-      char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10);
+      char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20);
       if( zDefn==0 ){
         rc = SQLITE_NOMEM;
       }else{
@@ -356,6 +381,14 @@ int sqlite3Fts5StorageOpen(
           sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
           iOff += (int)strlen(&zDefn[iOff]);
         }
+        if( pConfig->bLocale ){
+          for(i=0; i<pConfig->nCol; i++){
+            if( pConfig->abUnindexed[i]==0 ){
+              sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i);
+              iOff += (int)strlen(&zDefn[iOff]);
+            }
+          }
+        }
         rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
       }
       sqlite3_free(zDefn);
@@ -507,7 +540,8 @@ static int fts5StorageDeleteFromIndex(
       sqlite3_value *pVal = 0;
       const char *pText = 0;
       int nText = 0;
-      int bReset = 0;
+      const char *pLoc = 0;
+      int nLoc = 0;
 
       assert( pSeek==0 || apVal==0 );
       assert( pSeek!=0 || apVal!=0 );
@@ -517,10 +551,19 @@ static int fts5StorageDeleteFromIndex(
         pVal = apVal[iCol-1];
       }
 
-      rc = sqlite3Fts5ExtractText(
-          pConfig, pVal, pSeek!=0, &bReset, &pText, &nText
-      );
+      if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
+        rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+      }else{
+        pText = (const char*)sqlite3_value_text(pVal);
+        nText = sqlite3_value_bytes(pVal);
+        if( pConfig->bLocale && pSeek ){
+          pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol);
+          nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol);
+        }
+      }
+
       if( rc==SQLITE_OK ){
+        sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
         ctx.szCol = 0;
         rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
             pText, nText, (void*)&ctx, fts5StorageInsertCallback
@@ -529,7 +572,7 @@ static int fts5StorageDeleteFromIndex(
         if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
           rc = FTS5_CORRUPT;
         }
-        if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+        sqlite3Fts5ClearLocale(pConfig);
       }
     }
   }
@@ -788,20 +831,35 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
     for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
       ctx.szCol = 0;
       if( pConfig->abUnindexed[ctx.iCol]==0 ){
-        int bReset = 0;           /* True if tokenizer locale must be reset */
         int nText = 0;            /* Size of pText in bytes */
         const char *pText = 0;    /* Pointer to buffer containing text value */
+        int nLoc = 0;             /* Size of pLoc in bytes */
+        const char *pLoc = 0;     /* Pointer to buffer containing text value */
+
         sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
+        if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
+         && sqlite3Fts5IsLocaleValue(pConfig, pVal)
+        ){
+          rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+        }else{
+          pText = (const char*)sqlite3_value_text(pVal);
+          nText = sqlite3_value_bytes(pVal);
+          if( pConfig->bLocale ){
+            int iCol = ctx.iCol + 1 + pConfig->nCol;
+            pLoc = (const char*)sqlite3_column_text(pScan, iCol);
+            nLoc = sqlite3_column_bytes(pScan, iCol);
+          }
+        }
 
-        rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText);
         if( rc==SQLITE_OK ){
+          sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
           rc = sqlite3Fts5Tokenize(pConfig, 
               FTS5_TOKENIZE_DOCUMENT,
               pText, nText,
               (void*)&ctx,
               fts5StorageInsertCallback
           );
-          if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+          sqlite3Fts5ClearLocale(pConfig);
         }
       }
       sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
@@ -884,29 +942,45 @@ int sqlite3Fts5StorageContentInsert(
   }else{
     sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
     int i;                        /* Counter variable */
+    int nIndexed = 0;             /* Number indexed columns seen */
     rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
-    for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
+    if( pInsert ) sqlite3_clear_bindings(pInsert);
+
+    /* Bind the rowid value */
+    sqlite3_bind_value(pInsert, 1, apVal[1]);
+
+    /* Loop through values for user-defined columns. i=2 is the leftmost
+    ** user-defined column. As is column 1 of pSavedRow.  */
+    for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
+      int bUnindexed = pConfig->abUnindexed[i-2];
       sqlite3_value *pVal = apVal[i];
+
+      nIndexed += !bUnindexed;
       if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
         /* This is an UPDATE statement, and column (i-2) was not modified.
         ** Retrieve the value from Fts5Storage.pSavedRow instead. */
         pVal = sqlite3_column_value(p->pSavedRow, i-1);
-      }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){
+        if( pConfig->bLocale && bUnindexed==0 ){
+          sqlite3_bind_value(pInsert, pConfig->nCol + 1 + nIndexed,
+              sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1)
+          );
+        }
+      }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
+        const char *pText = 0;
+        const char *pLoc = 0;
+        int nText = 0;
+        int nLoc = 0;
         assert( pConfig->bLocale );
-        assert( i>1 );
-        if( pConfig->abUnindexed[i-2] ){
-          /* At attempt to insert an fts5_locale() value into an UNINDEXED
-          ** column. Strip the locale away and just bind the text.  */
-          const char *pText = 0;
-          int nText = 0;
-          rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
+
+        rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+        if( rc==SQLITE_OK ){
           sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
-        }else{
-          const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
-          int nBlob = sqlite3_value_bytes(pVal);
-          assert( nBlob>4 );
-          sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT);
+          if( bUnindexed==0 ){
+            int iLoc = pConfig->nCol + 1 + nIndexed;
+            sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT);
+          }
         }
+
         continue;
       }
 
@@ -945,23 +1019,37 @@ int sqlite3Fts5StorageIndexInsert(
   for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
     ctx.szCol = 0;
     if( pConfig->abUnindexed[ctx.iCol]==0 ){
-      int bReset = 0;             /* True if tokenizer locale must be reset */
       int nText = 0;              /* Size of pText in bytes */
       const char *pText = 0;      /* Pointer to buffer containing text value */
+      int nLoc = 0;               /* Size of pText in bytes */
+      const char *pLoc = 0;       /* Pointer to buffer containing text value */
+
       sqlite3_value *pVal = apVal[ctx.iCol+2];
-      int bDisk = 0;
       if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
         pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
-        bDisk = 1;
+        if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
+          int iCol = ctx.iCol + 1 + pConfig->nCol;
+          pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol);
+          nLoc = sqlite3_column_bytes(p->pSavedRow, iCol);
+        }
+      }else{
+        pVal = apVal[ctx.iCol+2];
+      }
+
+      if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
+        rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+      }else{
+        pText = (const char*)sqlite3_value_text(pVal);
+        nText = sqlite3_value_bytes(pVal);
       }
-      rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText);
+
       if( rc==SQLITE_OK ){
-        assert( bReset==0 || pConfig->bLocale );
+        sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
         rc = sqlite3Fts5Tokenize(pConfig, 
             FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
             fts5StorageInsertCallback
         );
-        if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+        sqlite3Fts5ClearLocale(pConfig);
       }
     }
     sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
@@ -1126,37 +1214,61 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
           rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
         }
         for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
-          if( pConfig->abUnindexed[i] ) continue;
-          ctx.iCol = i;
-          ctx.szCol = 0;
-          if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
-            rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
-          }
-          if( rc==SQLITE_OK ){
-            int bReset = 0;        /* True if tokenizer locale must be reset */
-            int nText = 0;         /* Size of pText in bytes */
-            const char *pText = 0; /* Pointer to buffer containing text value */
-
-            rc = sqlite3Fts5ExtractText(pConfig, 
-                sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText
-            );
+          if( pConfig->abUnindexed[i]==0 ){
+            const char *pText = 0;
+            int nText = 0;
+            const char *pLoc = 0;
+            int nLoc = 0;
+            sqlite3_value *pVal = sqlite3_column_value(pScan, i+1);
+
+            if( pConfig->eContent==FTS5_CONTENT_EXTERNAL 
+             && sqlite3Fts5IsLocaleValue(pConfig, pVal)
+            ){
+              rc = sqlite3Fts5DecodeLocaleValue(
+                  pVal, &pText, &nText, &pLoc, &nLoc
+              );
+            }else{
+              if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
+                int iCol = i + 1 + pConfig->nCol;
+                pLoc = (const char*)sqlite3_column_text(pScan, iCol);
+                nLoc = sqlite3_column_bytes(pScan, iCol);
+              }
+              pText = (const char*)sqlite3_value_text(pVal);
+              nText = sqlite3_value_bytes(pVal);
+            }
+
+            ctx.iCol = i;
+            ctx.szCol = 0;
+
+            if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+              rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
+            }
+
             if( rc==SQLITE_OK ){
+              sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
               rc = sqlite3Fts5Tokenize(pConfig, 
                   FTS5_TOKENIZE_DOCUMENT,
                   pText, nText,
                   (void*)&ctx,
                   fts5StorageIntegrityCallback
               );
-              if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+              sqlite3Fts5ClearLocale(pConfig);
+            }
+
+            /* If this is not a columnsize=0 database, check that the number
+            ** of tokens in the value matches the aColSize[] value read from
+            ** the %_docsize table.  */
+            if( rc==SQLITE_OK 
+             && pConfig->bColumnsize 
+             && ctx.szCol!=aColSize[i] 
+            ){
+              rc = FTS5_CORRUPT;
+            }
+            aTotalSize[i] += ctx.szCol;
+            if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+              sqlite3Fts5TermsetFree(ctx.pTermset);
+              ctx.pTermset = 0;
             }
-          }
-          if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
-            rc = FTS5_CORRUPT;
-          }
-          aTotalSize[i] += ctx.szCol;
-          if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
-            sqlite3Fts5TermsetFree(ctx.pTermset);
-            ctx.pTermset = 0;
           }
         }
         sqlite3Fts5TermsetFree(ctx.pTermset);
diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index 1e9f7bbb6..a8ab44096 100644
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -96,14 +96,14 @@ static int SQLITE_TCLAPI f5tDbAndApi(
 
     rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
     if( rc!=SQLITE_OK ){
-      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
       return TCL_ERROR;
     }
     sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
     sqlite3_step(pStmt);
 
     if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
-      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
       return TCL_ERROR;
     }
 
@@ -392,7 +392,7 @@ static int SQLITE_TCLAPI xF5tApi(
     CASE(12, "xSetAuxdata") {
       F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData));
       if( pData==0 ){
-        Tcl_AppendResult(interp, "out of memory", 0);
+        Tcl_AppendResult(interp, "out of memory", (char*)0);
         return TCL_ERROR;
       }
       pData->pObj = objv[2];
@@ -452,7 +452,7 @@ static int SQLITE_TCLAPI xF5tApi(
 
       rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
       if( rc!=SQLITE_OK ){
-        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0);
         return TCL_ERROR;
       }
       for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
@@ -683,7 +683,7 @@ static int SQLITE_TCLAPI f5tCreateFunction(
       pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy
   );
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
     return TCL_ERROR;
   }
 
@@ -750,7 +750,7 @@ static int SQLITE_TCLAPI f5tTokenize(
   if( objc==5 ){
     char *zOpt = Tcl_GetString(objv[1]);
     if( strcmp("-subst", zOpt) ){
-      Tcl_AppendResult(interp, "unrecognized option: ", zOpt, 0);
+      Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0);
       return TCL_ERROR;
     }
   }
@@ -759,7 +759,7 @@ static int SQLITE_TCLAPI f5tTokenize(
     return TCL_ERROR;
   }
   if( nArg==0 ){
-    Tcl_AppendResult(interp, "no such tokenizer: ", 0);
+    Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0);
     Tcl_Free((void*)azArg);
     return TCL_ERROR;
   }
@@ -767,13 +767,13 @@ static int SQLITE_TCLAPI f5tTokenize(
 
   rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], 0);
+    Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0);
     return TCL_ERROR;
   }
 
   rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
+    Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0);
     return TCL_ERROR;
   }
 
@@ -787,7 +787,7 @@ static int SQLITE_TCLAPI f5tTokenize(
   );
   tokenizer.xDelete(pTok);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0);
+    Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0);
     Tcl_DecrRefCount(pRet);
     return TCL_ERROR;
   }
@@ -1049,7 +1049,7 @@ static int SQLITE_TCLAPI f5tTokenizerLocale(
 
   if( p->xToken==0 ){
     Tcl_AppendResult(interp, 
-        "sqlite3_fts5_locale may only be used by tokenizer callback", 0
+        "sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0
     );
     return TCL_ERROR;
   }
@@ -1098,7 +1098,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
 
   if( p->xToken==0 ){
     Tcl_AppendResult(interp, 
-        "sqlite3_fts5_token may only be used by tokenizer callback", 0
+        "sqlite3_fts5_token may only be used by tokenizer callback", (char*)0
     );
     return TCL_ERROR;
   }
@@ -1250,7 +1250,7 @@ static int SQLITE_TCLAPI f5tCreateTokenizer(
     Tcl_AppendResult(interp, (
           bV2 ? "error in fts5_api.xCreateTokenizer_v2()"
           : "error in fts5_api.xCreateTokenizer()"
-    ), 0);
+    ), (char*)0);
     return TCL_ERROR;
   }
 
@@ -1540,13 +1540,93 @@ static int SQLITE_TCLAPI f5tRegisterOriginText(
 
   Tcl_ResetResult(interp);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
     return TCL_ERROR;
   }
   return TCL_OK;
 }
 
 /*
+** This function is used to DROP an fts5 table. It works even if the data
+** structures fts5 stores within the database are corrupt, which sometimes
+** prevents a straight "DROP TABLE" command from succeeding.
+**
+** The first parameter is the database handle to use for the DROP TABLE
+** operation. The second is the name of the database to drop the fts5 table
+** from (i.e. "main", "temp" or the name of an attached database). The
+** third parameter is the name of the fts5 table to drop.
+**
+** SQLITE_OK is returned if the table is successfully dropped. Or, if an
+** error occurs, an SQLite error code.
+*/
+static int sqlite3_fts5_drop_corrupt_table(
+  sqlite3 *db,                    /* Database handle */
+  const char *zDb,                /* Database name ("main", "temp" etc.) */
+  const char *zTab                /* Name of fts5 table to drop */
+){
+  int rc = SQLITE_OK;
+  int bDef = 0;
+
+  rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef);
+  if( rc==SQLITE_OK ){
+    char *zScript = sqlite3_mprintf(
+        "DELETE FROM %Q.'%q_data';"
+        "DELETE FROM %Q.'%q_config';"
+        "INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');"
+        "INSERT INTO %Q.'%q_config' VALUES('version', 4);"
+        "DROP TABLE %Q.'%q';",
+        zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab
+    );
+
+    if( zScript==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
+      rc = sqlite3_exec(db, zScript, 0, 0, 0);
+      if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0);
+      sqlite3_free(zScript);
+    }
+  }
+
+  return rc;
+}
+
+/*
+**      sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE
+**
+** Description...
+*/
+static int SQLITE_TCLAPI f5tDropCorruptTable(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db = 0;
+  const char *zDb = 0;
+  const char *zTab = 0;
+  int rc = SQLITE_OK;
+
+  if( objc!=4 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE");
+    return TCL_ERROR;
+  }
+  if( f5tDbPointer(interp, objv[1], &db) ){
+    return TCL_ERROR;
+  }
+  zDb = Tcl_GetString(objv[2]);
+  zTab = Tcl_GetString(objv[3]);
+
+  rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
+    return TCL_ERROR;
+  }
+
+  return TCL_OK;
+}
+
+/*
 ** Entry point.
 */
 int Fts5tcl_Init(Tcl_Interp *interp){
@@ -1564,7 +1644,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
     { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
     { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
     { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
-    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
+    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 },
+    { "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 }
   };
   int i;
   F5tTokenizerContext *pContext;
diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test
index 4233719fb..934855410 100644
--- a/ext/fts5/test/fts5blob.test
+++ b/ext/fts5/test/fts5blob.test
@@ -103,13 +103,13 @@ do_execsql_test 3.0 {
 
 do_catchsql_test 3.1 {
   INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456');
-} {1 {datatype mismatch}}
+} {0 {}}
 do_catchsql_test 3.2 {
   INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456');
-} {1 {datatype mismatch}}
+} {0 {}}
 do_catchsql_test 3.3 {
   INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456');
-} {1 {datatype mismatch}}
+} {0 {}}
 
 
 #--------------------------------------------------------------------------
diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test
index c5faaa87b..3e8b0377c 100644
--- a/ext/fts5/test/fts5corrupt3.test
+++ b/ext/fts5/test/fts5corrupt3.test
@@ -680,11 +680,11 @@ do_test 12.0 {
 
 do_catchsql_test 11.1 {
   SELECT * FROM t1 WHERE t1 MATCH 'abandon';
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}
 
 do_catchsql_test 11.2 {
   INSERT INTO t1(t1, rank) VALUES('merge', 500);
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}
 
 #-------------------------------------------------------------------------
 #
@@ -1040,7 +1040,7 @@ do_test 16.0 {
 
 do_catchsql_test 16.1 {
 INSERT INTO t1(t1) VALUES('integrity-check');
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}
 
 #--------------------------------------------------------------------------
 reset_db
@@ -1126,7 +1126,7 @@ do_test 17.0 {
 
 do_catchsql_test 17.1 {
   SELECT * FROM t1 WHERE t1 MATCH 'abandon';
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}
 
 #--------------------------------------------------------------------------
 reset_db
@@ -1630,7 +1630,7 @@ do_test 20.0 {
 
 do_catchsql_test 20.1 {
   SELECT * FROM t1 WHERE t1 MATCH 'abandon';
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}
 
 #-------------------------------------------------------------------------
 reset_db
@@ -2100,7 +2100,7 @@ do_test 22.0 {
 
 do_catchsql_test 22.1 {
   INSERT INTO t1(t1) VALUES('optimize');
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}
 
 #--------------------------------------------------------------------------
 reset_db
@@ -3700,7 +3700,7 @@ do_catchsql_test 32.1 {
          highlight(t1, 2, '[', ']') 
   FROM t1('g + h') 
   WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}
 
 do_catchsql_test 32.2 {
   SELECT * FROM t3;
diff --git a/ext/fts5/test/fts5corrupt8.test b/ext/fts5/test/fts5corrupt8.test
new file mode 100644
index 000000000..d642920e4
--- /dev/null
+++ b/ext/fts5/test/fts5corrupt8.test
@@ -0,0 +1,94 @@
+# 2024 Aug 28
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5corrupt8
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+}
+
+do_execsql_test 1.1 {
+  UPDATE t1_data SET block='hello world' WHERE id=10
+}
+
+db close
+sqlite3 db test.db
+
+do_catchsql_test 1.2 {
+  SELECT * FROM t1
+} {1 {database disk image is malformed}}
+do_catchsql_test 1.3 {
+  DROP TABLE t1
+} {0 {}}
+do_execsql_test 1.4 {
+  SELECT * FROM sqlite_schema
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+}
+do_execsql_test 2.1 {
+  UPDATE t1_config SET v=555 WHERE k='version'
+}
+db close
+sqlite3 db test.db
+do_catchsql_test 2.2 {
+  SELECT * FROM t1
+} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}}
+do_catchsql_test 2.3 {
+  DROP TABLE t1
+} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}}
+do_test 2.4 {
+  sqlite3_fts5_drop_corrupt_table db main t1
+} {}
+do_execsql_test 2.5 {
+  SELECT * FROM sqlite_schema
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+}
+do_execsql_test 3.1 {
+  DELETE FROM t1_config;
+}
+db close
+sqlite3 db test.db
+do_catchsql_test 3.2 {
+  SELECT * FROM t1
+} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}}
+do_catchsql_test 3.3 {
+  DROP TABLE t1
+} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}}
+
+
+do_test 3.4 {
+  sqlite3_db_config db DEFENSIVE 1
+} {1}
+do_test 3.5 {
+  sqlite3_fts5_drop_corrupt_table db main t1
+} {}
+do_test 3.6 {
+  sqlite3_db_config db DEFENSIVE -1
+} {1}
+do_execsql_test 3.7 {
+  SELECT * FROM sqlite_schema
+}
+
+finish_test
+
diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test
index 1d0d5c9b7..2b4f6c4d2 100644
--- a/ext/fts5/test/fts5fault4.test
+++ b/ext/fts5/test/fts5fault4.test
@@ -90,7 +90,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
 do_faultsim_test 4 -faults oom-* -body {
   db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
 } -test {
-  faultsim_test_result {0 {0 {} 3}}
+  faultsim_test_result {0 {0 {} 2}}
 }
 
 #-------------------------------------------------------------------------
diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test
index 63bfdd68a..72f25caee 100644
--- a/ext/fts5/test/fts5faultI.test
+++ b/ext/fts5/test/fts5faultI.test
@@ -246,7 +246,7 @@ do_execsql_test 10.1 {
 } {hello}
 
 faultsim_save_and_close
-do_faultsim_test 10 -faults oom* -prep {
+do_faultsim_test 10.1 -faults oom* -prep {
   faultsim_restore_and_reopen
 } -body {
   execsql {
@@ -256,6 +256,39 @@ do_faultsim_test 10 -faults oom* -prep {
   faultsim_test_result {0 hello}
 }
 
+breakpoint
+faultsim_save_and_close
+do_faultsim_test 10.2 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    INSERT INTO ft VALUES(zeroblob(10000));
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 11.0 {
+  CREATE VIRTUAL TABLE f1 USING fts5(content);
+  CREATE TABLE g1(id, content);
+  INSERT INTO g1 VALUES(30000, 'a b c');
+  INSERT INTO g1 VALUES(40000, 'd e f');
+}
+
+faultsim_save_and_close
+
+do_faultsim_test 11 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    INSERT INTO f1(rowid, content) SELECT id, content FROM g1;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
 
 finish_test
 
diff --git a/ext/fts5/test/fts5integrity2.test b/ext/fts5/test/fts5integrity2.test
new file mode 100644
index 000000000..968be3bdd
--- /dev/null
+++ b/ext/fts5/test/fts5integrity2.test
@@ -0,0 +1,56 @@
+# 2024 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests focused on the integrity-check procedure.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5integrity2
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(a, detail='none');
+  BEGIN;
+    INSERT INTO t2(rowid, a) VALUES(-1, 'hello world');
+    INSERT INTO t2(rowid, a) VALUES(9223372036854775807, 'hello world');
+  COMMIT;
+}
+
+do_execsql_test 2.1 {
+  SELECT rowid FROM t2('hello AND world');
+} {-1 9223372036854775807}
+
+#-------------------------------------------------------------------------
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none');
+  CREATE TABLE r1(r);
+  
+  WITH c(x) AS (VALUES(1) UNION SELECT x<<1 FROM c)
+    INSERT INTO r1(r) SELECT -1-x FROM c;
+
+  INSERT INTO t1(rowid, a) SELECT r, 'abc' FROM r1;
+}
+
+do_execsql_test 2.1 {
+  PRAGMA integrity_check;
+} {ok}
+
+do_execsql_test 2.2 {
+  SELECT rowid FROM t1('abc') ORDER BY +rowid;
+} [db eval {SELECT r FROM r1 ORDER BY r}]
+
+
+finish_test
diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test
index 684dcecd8..e5799fb7f 100644
--- a/ext/fts5/test/fts5locale.test
+++ b/ext/fts5/test/fts5locale.test
@@ -73,6 +73,7 @@ do_execsql_test 1.2 {
   SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') );
 } {2 cba}
 
+
 #-------------------------------------------------------------------------
 # Test that the locale= option exists and seems to accept values. And
 # that fts5_locale() values may only be inserted into an internal-content
@@ -99,8 +100,11 @@ do_catchsql_test 2.3 {
   INSERT INTO b1(b1, rank) VALUES('locale', 0);
 } {1 {SQL logic error}}
 
-do_execsql_test 2.4 {
+do_execsql_test 2.4.1 {
   INSERT INTO b1 VALUES('abc', 'one two three');
+}
+
+do_execsql_test 2.4.2 {
   INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six'));
 }
 
@@ -131,6 +135,7 @@ do_execsql_test 2.12  { SELECT quote(y) FROM b1('ruof') } {
 do_execsql_test 2.13 {
   INSERT INTO b1(b1) VALUES('integrity-check');
 }
+
 do_execsql_test 2.14 {
   INSERT INTO b1(b1) VALUES('rebuild');
 }
@@ -149,7 +154,6 @@ do_execsql_test 2.18 {
   INSERT INTO b1(rowid, x, y) VALUES(
       test_setsubtype(45, 76), 'abc def', 'def abc'
   );
-  INSERT INTO b1(b1) VALUES('integrity-check');
 }
 
 #-------------------------------------------------------------------------
@@ -278,9 +282,9 @@ do_execsql_test 5.2 {
 }
 
 do_execsql_test 5.3 {
-  SELECT typeof(c0), typeof(c1) FROM t1_content
+  SELECT typeof(c0), typeof(c1), typeof(l0) FROM t1_content
 } {
-  blob text
+  text text text
 }
 
 #-------------------------------------------------------------------------
@@ -305,37 +309,37 @@ foreach {tn opt} {
 
   fts5_aux_test_functions db
 
-  do_execsql_test 5.$tn.3 {
+  do_execsql_test 6.$tn.3 {
     SELECT fts5_test_columnsize(y1) FROM y1
   } {
     2 3 2 4
   }
 
-  do_execsql_test 5.$tn.4 {
+  do_execsql_test 6.$tn.4 {
     SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
   } {
     2 3
   }
 
-  do_execsql_test 5.$tn.5 {
+  do_execsql_test 6.$tn.5 {
     SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
   } {
     2 3
   }
 
-  do_execsql_test 5.$tn.6 {
+  do_execsql_test 6.$tn.6 {
     SELECT rowid, fts5_test_columnsize(y1) FROM y1('have');
   } {
     4 4
   }
 
-  do_execsql_test 5.$tn.7 {
+  do_execsql_test 6.$tn.7 {
     SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have');
   } {
     4 {which it hath been used to [have]}
   }
 
-  do_execsql_test 5.$tn.8 {
+  do_execsql_test 6.$tn.8 {
     SELECT rowid, 
            highlight(y1, 0, '[', ']'),
            snippet(y1, 0, '[', ']', '...', 10)
@@ -473,7 +477,7 @@ foreach_detail_mode $::testprefix {
   }
   
   foreach {tn v} {
-    1   X'001122'
+    1   X'001152'
     2   X'0011223344'
     3   X'00E0B2EB68656c6c6f'
     4   X'00E0B2EB0068656c6c6f'
@@ -484,33 +488,33 @@ foreach_detail_mode $::testprefix {
   
     do_catchsql_test 10.2.$tn.3 {
       INSERT INTO ft(ft) VALUES('rebuild');
-    } {1 {SQL logic error}}
+    } {0 {}}
   
     do_catchsql_test 10.2.$tn.4 "
       SELECT * FROM ft( test_setsubtype($v, 76) );
-    " {1 {SQL logic error}}
+    " {1 {fts5: syntax error near ""}}
   
     do_execsql_test 10.2.$tn.5 {
       INSERT INTO ft(rowid, x) VALUES(1, 'hello world');
     }
   
-    if {"%DETAIL%"!="full"} {
-      do_catchsql_test 10.2.$tn.6 {
+    if {"%DETAIL%"=="full"} {
+      do_execsql_test 10.2.$tn.6 {
         SELECT fts5_test_poslist(ft) FROM ft('world');
-      } {1 SQLITE_ERROR}
+      } {0.0.1}
   
-      do_catchsql_test 10.2.$tn.7 {
+      do_execsql_test 10.2.$tn.7.1 {
         SELECT fts5_test_columnsize(ft) FROM ft('world');
-      } {1 SQLITE_ERROR}
+      } {1}
 
-      do_catchsql_test 10.2.$tn.7 {
+      do_execsql_test 10.2.$tn.7.2 {
         SELECT fts5_test_columnlocale(ft) FROM ft('world');
-      } {1 SQLITE_ERROR}
+      } {{{}}}
     }
 
     do_catchsql_test 10.2.$tn.8 {
-      SELECT * FROM ft('hello')
-    } {1 {SQL logic error}}
+      SELECT count(*) FROM ft('hello')
+    } {0 1}
 
     do_catchsql_test 10.2.$tn.9 {
       PRAGMA integrity_check;
@@ -523,11 +527,11 @@ foreach_detail_mode $::testprefix {
 
     do_catchsql_test 10.2.$tn.11 "
       INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) )
-    " {1 {SQL logic error}}
+    " {0 {}}
 
     do_catchsql_test 10.2.$tn.12 "
       INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) )
-    " {1 {SQL logic error}}
+    " {0 {}}
 
     do_execsql_test 10.2.$tn.13 {
       INSERT INTO ft2(rowid, x) VALUES(1, 'hello world');
@@ -536,7 +540,7 @@ foreach_detail_mode $::testprefix {
 
     do_catchsql_test 10.2.$tn.15 {
       PRAGMA integrity_check;
-    } {1 {SQL logic error}}
+    } {0 {{malformed inverted index for FTS5 table main.ft2}}}
 
     do_execsql_test 10.2.$tn.16 {
       DELETE FROM ft2_content;
@@ -663,5 +667,82 @@ do_catchsql_test 13.2.7 {
   FROM ft('one AND three') ORDER BY rowid
 } {1 {non-integer argument passed to function fts5_get_locale()}}
 
+#-------------------------------------------------------------------------
+# Check that UPDATE statements that may affect more than one row work.
+#
+reset_db
+do_execsql_test 14.1 {
+  CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1);
+}
+
+do_execsql_test 14.2 {
+  INSERT INTO ft VALUES('hello', 'world');
+}
+
+do_execsql_test 14.3 {
+  UPDATE ft SET b = fts5_locale('en_AU', 'world');
+}
+
+do_execsql_test 14.4 {
+  INSERT INTO ft VALUES(X'abcd', X'1234');
+} {}
+
+do_execsql_test 14.5 {
+  SELECT quote(a), quote(b) FROM ft
+} {'hello' 'world' X'ABCD' X'1234'}
+
+do_execsql_test 14.6 {
+  DELETE FROM ft;
+  INSERT INTO ft VALUES(NULL, 'null');
+  INSERT INTO ft VALUES(123, 'int');
+  INSERT INTO ft VALUES(345.0, 'real');
+  INSERT INTO ft VALUES('abc', 'text');
+  INSERT INTO ft VALUES(fts5_locale('abc', 'def'), 'text');
+
+  SELECT a, typeof(a), b FROM ft
+} {
+  {} null null
+  123 integer int
+  345.0 real real
+  abc text text
+  def text text
+}
+
+do_execsql_test 14.7 {
+  SELECT quote(c0), typeof(c0) FROM ft_content
+} {
+  NULL null
+  123 integer
+  345.0 real
+  'abc' text
+  'def' text
+}
+
+#-------------------------------------------------------------------------
+# Check that inserting UNINDEXED columns between indexed columns of a
+# locale=1 table does not cause a problem.
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+fts5_aux_test_functions db
+
+do_execsql_test 15.1 {
+  CREATE VIRTUAL TABLE ft USING fts5(a, b UNINDEXED, c, locale=1, tokenize=tcl);
+}
+
+do_execsql_test 15.2 {
+  INSERT INTO ft VALUES('one', 'two', 'three');
+  INSERT INTO ft VALUES('one', 'two', fts5_locale('loc', 'three'));
+}
+
+do_execsql_test 15.3 {
+  SELECT c2, l2 FROM ft_content
+} {three {} three loc}
+
+do_execsql_test 15.4 {
+  SELECT c, fts5_columnlocale(ft, 2) FROM ft
+} {three {} three loc}
+
+
 finish_test
 
diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test
index 534c42fff..c2e580c56 100644
--- a/ext/fts5/test/fts5misc.test
+++ b/ext/fts5/test/fts5misc.test
@@ -44,12 +44,12 @@ do_catchsql_test 1.2.2 {
 
 do_catchsql_test 1.3.1 { 
   SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'); 
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 0}}
 
 do_catchsql_test 1.3.2 {
   SELECT a FROM t1
     WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 0}}
 
 db close
 sqlite3 db test.db
@@ -57,12 +57,12 @@ sqlite3 db test.db
 do_catchsql_test 1.3.3 {
   SELECT a FROM t1
     WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 0}}
 
 fts5_aux_test_functions db
 do_catchsql_test 1.3.4 {
   SELECT fts5_columntext(t1) FROM t1('*reads');
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 0}}
 
 #-------------------------------------------------------------------------
 reset_db
diff --git a/ext/fts5/test/fts5phrase.test b/ext/fts5/test/fts5phrase.test
index ea425a4dd..708cdfd83 100644
--- a/ext/fts5/test/fts5phrase.test
+++ b/ext/fts5/test/fts5phrase.test
@@ -93,15 +93,21 @@ foreach {tn cols tokens} {
   10 {b}      "i e"
   11 {a}      "i e"
 } {
-  set fts   "{$cols}:[join $tokens +]"
   set where [list]
   foreach c $cols { lappend where "pmatch($c, '$tokens')" }
   set where [join $where " OR "]
 
-  set res [db eval "SELECT rowid FROM t3 WHERE $where"]
-  do_execsql_test "1.$tn.$fts->([llength $res] rows)" { 
-    SELECT rowid FROM t3($fts) 
-  } $res
+  foreach fts [list \
+      "{$cols}:[join $tokens +]"       \
+      "{$cols}:NEAR([join $tokens +])" \
+      "{$cols}:NEAR([join $tokens +],1)" \
+      "{$cols}:NEAR([join $tokens +],111)" \
+  ] {
+    set res [db eval "SELECT rowid FROM t3 WHERE $where"]
+    do_execsql_test "1.$tn.$fts->([llength $res] rows)" { 
+      SELECT rowid FROM t3($fts) 
+    } $res
+  }
 }
 
 do_execsql_test 2.0 {
diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test
index 60ccb5a9c..ad59bf0d9 100644
--- a/ext/fts5/test/fts5simple.test
+++ b/ext/fts5/test/fts5simple.test
@@ -350,7 +350,7 @@ do_execsql_test 14.3 {
 
 do_execsql_test 14.4 {
   SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
-} {0 {} 3}
+} {0 {} 2}
 
 #-------------------------------------------------------------------------
 reset_db
diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
index ca8090ed2..c2ecab0b2 100644
--- a/ext/misc/fileio.c
+++ b/ext/misc/fileio.c
@@ -40,7 +40,7 @@
 **   modification-time of the target file is set to this value before
 **   returning.
 **
-**   If three or more arguments are passed to this function and an
+**   If five or more arguments are passed to this function and an
 **   error is encountered, an exception is raised.
 **
 ** READFILE(FILE):
diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c
index cccbaf025..1c6191d42 100644
--- a/ext/misc/percentile.c
+++ b/ext/misc/percentile.c
@@ -11,7 +11,7 @@
 ******************************************************************************
 **
 ** This file contains code to implement the percentile(Y,P) SQL function
-** as described below:
+** and similar as described below:
 **
 **   (1)  The percentile(Y,P) function is an aggregate function taking
 **        exactly two arguments.
@@ -60,31 +60,105 @@
 **
 **  (13)  A separate median(Y) function is the equivalent percentile(Y,50).
 **
-**  (14)  A separate percentile_cond(Y,X) function is the equivalent of
-**        percentile(Y,X*100.0).
+**  (14)  A separate percentile_cont(Y,P) function is equivalent to
+**        percentile(Y,P/100.0).  In other words, the fraction value in
+**        the second argument is in the range of 0 to 1 instead of 0 to 100.
+**
+**  (15)  A separate percentile_disc(Y,P) function is like
+**        percentile_cont(Y,P) except that instead of returning the weighted
+**        average of the nearest two input values, it returns the next lower
+**        value.  So the percentile_disc(Y,P) will always return a value
+**        that was one of the inputs.
+**
+**  (16)  All of median(), percentile(Y,P), percentile_cont(Y,P) and
+**        percentile_disc(Y,P) can be used as window functions.
+**
+** Differences from standard SQL:
+**
+**  *  The percentile_cont(X,P) function is equivalent to the following in
+**     standard SQL:
+**
+**         (percentile_cont(P) WITHIN GROUP (ORDER BY X))
+**
+**     The SQLite syntax is much more compact.  The standard SQL syntax
+**     is also supported if SQLite is compiled with the
+**     -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option.
+**
+**  *  No median(X) function exists in the SQL standard.  App developers
+**     are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)".
+**
+**  *  No percentile(Y,P) function exists in the SQL standard.  Instead of
+**     percential(Y,P), developers must write this:
+**     "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)".  Note that
+**     the fraction parameter to percentile() goes from 0 to 100 whereas
+**     the fraction parameter in SQL standard percentile_cont() goes from
+**     0 to 1.
+**
+** Implementation notes as of 2024-08-31:
+**
+**  *  The regular aggregate-function versions of these routines work
+**     by accumulating all values in an array of doubles, then sorting
+**     that array using quicksort before computing the answer. Thus
+**     the runtime is O(NlogN) where N is the number of rows of input.
+**
+**  *  For the window-function versions of these routines, the array of
+**     inputs is sorted as soon as the first value is computed.  Thereafter,
+**     the array is kept in sorted order using an insert-sort.  This
+**     results in O(N*K) performance where K is the size of the window.
+**     One can imagine alternative implementations that give O(N*logN*logK)
+**     performance, but they require more complex logic and data structures.
+**     The developers have elected to keep the asymptotically slower
+**     algorithm for now, for simplicity, under the theory that window
+**     functions are seldom used and when they are, the window size K is
+**     often small.  The developers might revisit that decision later,
+**     should the need arise.
 */
-#include "sqlite3ext.h"
-SQLITE_EXTENSION_INIT1
+#if defined(SQLITE3_H)
+  /* no-op */
+#elif defined(SQLITE_STATIC_PERCENTILE)
+#  include "sqlite3.h"
+#else
+#  include "sqlite3ext.h"
+   SQLITE_EXTENSION_INIT1
+#endif
 #include <assert.h>
 #include <string.h>
 #include <stdlib.h>
 
-/* The following object is the session context for a single percentile()
-** function.  We have to remember all input Y values until the very end.
+/* The following object is the group context for a single percentile()
+** aggregate.  Remember all input Y values until the very end.
 ** Those values are accumulated in the Percentile.a[] array.
 */
 typedef struct Percentile Percentile;
 struct Percentile {
   unsigned nAlloc;     /* Number of slots allocated for a[] */
   unsigned nUsed;      /* Number of slots actually used in a[] */
-  double rPct;         /* 1.0 more than the value for P */
+  char bSorted;        /* True if a[] is already in sorted order */
+  char bKeepSorted;    /* True if advantageous to keep a[] sorted */
+  char bPctValid;      /* True if rPct is valid */
+  double rPct;         /* Fraction.  0.0 to 1.0 */
   double *a;           /* Array of Y values */
 };
 
+/* Details of each function in the percentile family */
+typedef struct PercentileFunc PercentileFunc;
+struct PercentileFunc {
+  const char *zName;   /* Function name */
+  char nArg;           /* Number of arguments */
+  char mxFrac;         /* Maximum value of the "fraction" input */
+  char bDiscrete;      /* True for percentile_disc() */
+};
+static const PercentileFunc aPercentFunc[] = {
+  { "median",           1,   1, 0 },
+  { "percentile",       2, 100, 0 },
+  { "percentile_cont",  2,   1, 0 },
+  { "percentile_disc",  2,   1, 1 },
+};
+
 /*
 ** Return TRUE if the input floating-point number is an infinity.
 */
-static int isInfinity(double r){
+static int percentIsInfinity(double r){
   sqlite3_uint64 u;
   assert( sizeof(u)==sizeof(r) );
   memcpy(&u, &r, sizeof(u));
@@ -92,14 +166,65 @@ static int isInfinity(double r){
 }
 
 /*
-** Return TRUE if two doubles differ by 0.001 or less
+** Return TRUE if two doubles differ by 0.001 or less.
 */
-static int sameValue(double a, double b){
+static int percentSameValue(double a, double b){
   a -= b;
   return a>=-0.001 && a<=0.001;
 }
 
 /*
+** Search p (which must have p->bSorted) looking for an entry with
+** value y.  Return the index of that entry.
+**
+** If bExact is true, return -1 if the entry is not found.
+**
+** If bExact is false, return the index at which a new entry with
+** value y should be insert in order to keep the values in sorted
+** order.  The smallest return value in this case will be 0, and
+** the largest return value will be p->nUsed.
+*/
+static int percentBinarySearch(Percentile *p, double y, int bExact){
+  int iFirst = 0;              /* First element of search range */
+  int iLast = p->nUsed - 1;    /* Last element of search range */
+  while( iLast>=iFirst ){
+    int iMid = (iFirst+iLast)/2;
+    double x = p->a[iMid];
+    if( x<y ){
+      iFirst = iMid + 1;
+    }else if( x>y ){
+      iLast = iMid - 1;
+    }else{
+      return iMid;
+    }
+  }
+  if( bExact ) return -1;
+  return iFirst;
+}
+
+/*
+** Generate an error for a percentile function.
+**
+** The error format string must have exactly one occurrance of "%%s()"
+** (with two '%' characters).  That substring will be replaced by the name
+** of the function.
+*/
+static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){
+  PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx);
+  char *zMsg1;
+  char *zMsg2;
+  va_list ap;
+
+  va_start(ap, zFormat);
+  zMsg1 = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, pFunc->zName) : 0;
+  sqlite3_result_error(pCtx, zMsg2, -1);
+  sqlite3_free(zMsg1);
+  sqlite3_free(zMsg2);
+}
+
+/*
 ** The "step" function for percentile(Y,P) is called once for each
 ** input row.
 */
@@ -112,28 +237,20 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
 
   if( argc==1 ){
     /* Requirement 13:  median(Y) is the same as percentile(Y,50). */
-    rPct = 50.0;
-  }else if( sqlite3_user_data(pCtx)==0 ){
-    /* Requirement 3:  P must be a number between 0 and 100 */
-    eType = sqlite3_value_numeric_type(argv[1]);
-    rPct = sqlite3_value_double(argv[1]);
-    if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT)
-     || rPct<0.0 || rPct>100.0 ){
-       sqlite3_result_error(pCtx, "2nd argument to percentile() is not "
-                           "a number between 0.0 and 100.0", -1);
-      return;
-    }
+    rPct = 0.5;
   }else{
-    /* Requirement 3:  P must be a number between 0 and 1 */
+    /* Requirement 3:  P must be a number between 0 and 100 */
+    PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx);
     eType = sqlite3_value_numeric_type(argv[1]);
-    rPct = sqlite3_value_double(argv[1]);
+    rPct = sqlite3_value_double(argv[1])/(double)pFunc->mxFrac;
     if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT)
-     || rPct<0.0 || rPct>1.0 ){
-       sqlite3_result_error(pCtx, "2nd argument to percentile_cont() is not "
-                           "a number between 0.0 and 1.0", -1);
+     || rPct<0.0 || rPct>1.0
+    ){
+      percentError(pCtx, "the fraction argument to %%s()"
+                        " is not between 0.0 and %.1f",
+                        (double)pFunc->mxFrac);
       return;
     }
-    rPct *= 100.0;
   }
 
   /* Allocate the session context. */
@@ -142,11 +259,12 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
 
   /* Remember the P value.  Throw an error if the P value is different
   ** from any prior row, per Requirement (2). */
-  if( p->rPct==0.0 ){
-    p->rPct = rPct+1.0;
-  }else if( !sameValue(p->rPct,rPct+1.0) ){
-    sqlite3_result_error(pCtx, "2nd argument to percentile() is not the "
-                               "same for all input rows", -1);
+  if( !p->bPctValid ){
+    p->rPct = rPct;
+    p->bPctValid = 1;
+  }else if( !percentSameValue(p->rPct,rPct) ){
+    percentError(pCtx, "the fraction argument to %%s()"
+                      " is not the same for all input rows");
     return;
   }
 
@@ -157,15 +275,14 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
   /* If not NULL, then Y must be numeric.  Otherwise throw an error.
   ** Requirement 4 */
   if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){
-    sqlite3_result_error(pCtx, "1st argument to percentile() is not "
-                               "numeric", -1);
+    percentError(pCtx, "input to %%s() is not numeric");
     return;
   }
 
   /* Throw an error if the Y value is infinity or NaN */
   y = sqlite3_value_double(argv[0]);
-  if( isInfinity(y) ){
-    sqlite3_result_error(pCtx, "Inf input to percentile()", -1);
+  if( percentIsInfinity(y) ){
+    percentError(pCtx, "Inf input to %%s()");
     return;
   }
 
@@ -182,50 +299,80 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
     p->nAlloc = n;
     p->a = a;
   }
-  p->a[p->nUsed++] = y;
+  if( p->nUsed==0 ){
+    p->a[p->nUsed++] = y;
+    p->bSorted = 1;
+  }else if( !p->bSorted || y>=p->a[p->nUsed-1] ){
+    p->a[p->nUsed++] = y;
+  }else if( p->bKeepSorted ){
+    int i;
+    i = percentBinarySearch(p, y, 0);
+    if( i<(int)p->nUsed ){
+      memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0]));
+    }
+    p->a[i] = y;
+    p->nUsed++;
+  }else{
+    p->a[p->nUsed++] = y;
+    p->bSorted = 0;
+  }
 }
 
 /*
+** Interchange two doubles.
+*/
+#define SWAP_DOUBLE(X,Y)  {double ttt=(X);(X)=(Y);(Y)=ttt;}
+
+/*
 ** Sort an array of doubles.
+**
+** Algorithm: quicksort
+**
+** This is implemented separately rather than using the qsort() routine
+** from the standard library because:
+**
+**    (1)  To avoid a dependency on qsort()
+**    (2)  To avoid the function call to the comparison routine for each
+**         comparison.
 */
-static void sortDoubles(double *a, int n){
-  int iLt;       /* Entries with index less than iLt are less than rPivot */
-  int iGt;       /* Entries with index iGt or more are greater than rPivot */
+static void percentSort(double *a, unsigned int n){
+  int iLt;  /* Entries before a[iLt] are less than rPivot */
+  int iGt;  /* Entries at or after a[iGt] are greater than rPivot */
   int i;         /* Loop counter */
   double rPivot; /* The pivot value */
-  double rTmp;   /* Temporary used to swap two values */
-
-  if( n<2 ) return;
-  if( n>5 ){
-    rPivot = (a[0] + a[n/2] + a[n-1])/3.0;
-  }else{
-    rPivot = a[n/2];
+  
+  assert( n>=2 );
+  if( a[0]>a[n-1] ){
+    SWAP_DOUBLE(a[0],a[n-1])
+  }
+  if( n==2 ) return;
+  iGt = n-1;
+  i = n/2;
+  if( a[0]>a[i] ){
+    SWAP_DOUBLE(a[0],a[i])
+  }else if( a[i]>a[iGt] ){
+    SWAP_DOUBLE(a[i],a[iGt])
   }
-  iLt = i = 0;
-  iGt = n;
-  while( i<iGt ){
+  if( n==3 ) return;
+  rPivot = a[i];
+  iLt = i = 1;
+  do{
     if( a[i]<rPivot ){
-      if( i>iLt ){
-        rTmp = a[i];
-        a[i] = a[iLt];
-        a[iLt] = rTmp;
-      }
+      if( i>iLt ) SWAP_DOUBLE(a[i],a[iLt])
       iLt++;
       i++;
     }else if( a[i]>rPivot ){
       do{
         iGt--;
       }while( iGt>i && a[iGt]>rPivot );
-      rTmp = a[i];
-      a[i] = a[iGt];
-      a[iGt] = rTmp;
+      SWAP_DOUBLE(a[i],a[iGt])
     }else{
       i++;
     }
-  }
-  if( iLt>=2 ) sortDoubles(a, iLt);
-  if( n-iGt>=2 ) sortDoubles(a+iGt, n-iGt);
-
+  }while( i<iGt );
+  if( iLt>=2 ) percentSort(a, iLt);
+  if( n-iGt>=2 ) percentSort(a+iGt, n-iGt);
+    
 /* Uncomment for testing */
 #if 0
   for(i=0; i<n-1; i++){
@@ -234,12 +381,61 @@ static void sortDoubles(double *a, int n){
 #endif
 }
 
+
 /*
-** Called to compute the final output of percentile() and to clean
-** up all allocated memory.
+** The "inverse" function for percentile(Y,P) is called to remove a
+** row that was previously inserted by "step".
 */
-static void percentFinal(sqlite3_context *pCtx){
+static void percentInverse(sqlite3_context *pCtx,int argc,sqlite3_value **argv){
+  Percentile *p;
+  int eType;
+  double y;
+  int i;
+  assert( argc==2 || argc==1 );
+
+  /* Allocate the session context. */
+  p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p));
+  assert( p!=0 );
+
+  /* Ignore rows for which Y is NULL */
+  eType = sqlite3_value_type(argv[0]);
+  if( eType==SQLITE_NULL ) return;
+
+  /* If not NULL, then Y must be numeric.  Otherwise throw an error.
+  ** Requirement 4 */
+  if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){
+    return;
+  }
+
+  /* Ignore the Y value if it is infinity or NaN */
+  y = sqlite3_value_double(argv[0]);
+  if( percentIsInfinity(y) ){
+    return;
+  }
+  if( p->bSorted==0 ){
+    assert( p->nUsed>1 );
+    percentSort(p->a, p->nUsed);
+    p->bSorted = 1;
+  }
+  p->bKeepSorted = 1;
+
+  /* Find and remove the row */
+  i = percentBinarySearch(p, y, 1);
+  if( i>=0 ){
+    p->nUsed--;
+    if( i<(int)p->nUsed ){
+      memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0]));
+    }
+  }
+}
+
+/*
+** Compute the final output of percentile().  Clean up all allocated
+** memory if and only if bIsFinal is true.
+*/
+static void percentCompute(sqlite3_context *pCtx, int bIsFinal){
   Percentile *p;
+  PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx);
   unsigned i1, i2;
   double v1, v2;
   double ix, vx;
@@ -247,21 +443,38 @@ static void percentFinal(sqlite3_context *pCtx){
   if( p==0 ) return;
   if( p->a==0 ) return;
   if( p->nUsed ){
-    sortDoubles(p->a, p->nUsed);
-    ix = (p->rPct-1.0)*(p->nUsed-1)*0.01;
+    if( p->bSorted==0 ){
+      assert( p->nUsed>1 );
+      percentSort(p->a, p->nUsed);
+      p->bSorted = 1;
+    }
+    ix = p->rPct*(p->nUsed-1);
     i1 = (unsigned)ix;
-    i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1;
-    v1 = p->a[i1];
-    v2 = p->a[i2];
-    vx = v1 + (v2-v1)*(ix-i1);
+    if( pFunc->bDiscrete ){
+      vx = p->a[i1];
+    }else{
+      i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1;
+      v1 = p->a[i1];
+      v2 = p->a[i2];
+      vx = v1 + (v2-v1)*(ix-i1);
+    }
     sqlite3_result_double(pCtx, vx);
   }
-  sqlite3_free(p->a);
-  memset(p, 0, sizeof(*p));
+  if( bIsFinal ){
+    sqlite3_free(p->a);
+    memset(p, 0, sizeof(*p));
+  }else{
+    p->bKeepSorted = 1;
+  }
+}
+static void percentFinal(sqlite3_context *pCtx){
+  percentCompute(pCtx, 1);
+}
+static void percentValue(sqlite3_context *pCtx){
+  percentCompute(pCtx, 0);
 }
 
-
-#ifdef _WIN32
+#if defined(_WIN32) && !defined(SQLITE3_H) && !defined(SQLITE_STATIC_PERCENTILE)
 __declspec(dllexport)
 #endif
 int sqlite3_percentile_init(
@@ -270,20 +483,21 @@ int sqlite3_percentile_init(
   const sqlite3_api_routines *pApi
 ){
   int rc = SQLITE_OK;
+  unsigned int i;
+#if defined(SQLITE3_H) || defined(SQLITE_STATIC_PERCENTILE)
+  (void)pApi;      /* Unused parameter */
+#else
   SQLITE_EXTENSION_INIT2(pApi);
+#endif
   (void)pzErrMsg;  /* Unused parameter */
-  rc = sqlite3_create_function(db, "percentile", 2, 
-                               SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
-                               0, percentStep, percentFinal);
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "median", 1, 
-                                 SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
-                                 0, percentStep, percentFinal);
-  }
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(db, "percentile_cont", 2, 
-                                 SQLITE_UTF8|SQLITE_INNOCUOUS, &percentStep,
-                                 0, percentStep, percentFinal);
+  for(i=0; i<sizeof(aPercentFunc)/sizeof(aPercentFunc[0]); i++){
+    rc = sqlite3_create_window_function(db,
+            aPercentFunc[i].zName,
+            aPercentFunc[i].nArg,
+            SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_SELFORDER1,
+            (void*)&aPercentFunc[i],
+            percentStep, percentFinal, percentValue, percentInverse, 0);
+    if( rc ) break;
   }
   return rc;
 }
diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c
index 9790a1d87..07d797060 100644
--- a/ext/misc/sha1.c
+++ b/ext/misc/sha1.c
@@ -196,7 +196,8 @@ static void hash_step_vformat(
 ** zOut[].  zOut[] must be at least 41 bytes long. */
 static void hash_finish(
   SHA1Context *p,           /* The SHA1 context to finish and render */
-  char *zOut                /* Store hexadecimal hash here */
+  char *zOut,               /* Store hex or binary hash here */
+  int bAsBinary             /* 1 for binary hash, 0 for hex hash */
 ){
   unsigned int i;
   unsigned char finalcount[8];
@@ -215,11 +216,15 @@ static void hash_finish(
   for (i = 0; i < 20; i++){
     digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
   }
-  for(i=0; i<20; i++){
-    zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
-    zOut[i*2+1] = zEncode[digest[i] & 0xf];
+  if( bAsBinary ){
+    memcpy(zOut, digest, 20);
+  }else{
+    for(i=0; i<20; i++){
+      zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
+      zOut[i*2+1] = zEncode[digest[i] & 0xf];
+    }
+    zOut[i*2]= 0;
   }
-  zOut[i*2]= 0;
 }
 /* End of the hashing logic
 *****************************************************************************/
@@ -251,8 +256,13 @@ static void sha1Func(
   }else{
     hash_step(&cx, sqlite3_value_text(argv[0]), nByte);
   }
-  hash_finish(&cx, zOut);
-  sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT);
+  if( sqlite3_user_data(context)!=0 ){
+    hash_finish(&cx, zOut, 1);
+    sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT);
+  }else{
+    hash_finish(&cx, zOut, 0);
+    sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT);
+  }
 }
 
 /*
@@ -365,7 +375,7 @@ static void sha1QueryFunc(
     }
     sqlite3_finalize(pStmt);
   }
-  hash_finish(&cx, zOut);
+  hash_finish(&cx, zOut, 0);
   sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT);
 }
 
@@ -379,11 +389,17 @@ int sqlite3_sha_init(
   const sqlite3_api_routines *pApi
 ){
   int rc = SQLITE_OK;
+  static int one = 1;
   SQLITE_EXTENSION_INIT2(pApi);
   (void)pzErrMsg;  /* Unused parameter */
   rc = sqlite3_create_function(db, "sha1", 1, 
                        SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC,
-                               0, sha1Func, 0, 0);
+                                0, sha1Func, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "sha1b", 1, 
+                       SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC,
+                          (void*)&one, sha1Func, 0, 0);
+  }
   if( rc==SQLITE_OK ){
     rc = sqlite3_create_function(db, "sha1_query", 1, 
                                  SQLITE_UTF8|SQLITE_DIRECTONLY, 0,
diff --git a/ext/misc/vfstrace.c b/ext/misc/vfstrace.c
new file mode 100644
index 000000000..b7c8175ee
--- /dev/null
+++ b/ext/misc/vfstrace.c
@@ -0,0 +1,987 @@
+/*
+** 2011 March 16
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code implements a VFS shim that writes diagnostic
+** output for each VFS call, similar to "strace".
+**
+** USAGE:
+**
+** This source file exports a single symbol which is the name of a
+** function:
+**
+**   int vfstrace_register(
+**     const char *zTraceName,         // Name of the newly constructed VFS
+**     const char *zOldVfsName,        // Name of the underlying VFS
+**     int (*xOut)(const char*,void*), // Output routine.  ex: fputs
+**     void *pOutArg,                  // 2nd argument to xOut.  ex: stderr
+**     int makeDefault                 // Make the new VFS the default
+**   );
+**
+** Applications that want to trace their VFS usage must provide a callback
+** function with this prototype:
+**
+**   int traceOutput(const char *zMessage, void *pAppData);
+**
+** This function will "output" the trace messages, where "output" can
+** mean different things to different applications.  The traceOutput function
+** for the command-line shell (see shell.c) is "fputs" from the standard
+** library, which means that all trace output is written on the stream
+** specified by the second argument.  In the case of the command-line shell
+** the second argument is stderr.  Other applications might choose to output
+** trace information to a file, over a socket, or write it into a buffer.
+**
+** The vfstrace_register() function creates a new "shim" VFS named by
+** the zTraceName parameter.  A "shim" VFS is an SQLite backend that does
+** not really perform the duties of a true backend, but simply filters or
+** interprets VFS calls before passing them off to another VFS which does
+** the actual work.  In this case the other VFS - the one that does the
+** real work - is identified by the second parameter, zOldVfsName.  If
+** the 2nd parameter is NULL then the default VFS is used.  The common
+** case is for the 2nd parameter to be NULL.
+**
+** The third and fourth parameters are the pointer to the output function
+** and the second argument to the output function.  For the SQLite
+** command-line shell, when the -vfstrace option is used, these parameters
+** are fputs and stderr, respectively.
+**
+** The fifth argument is true (non-zero) to cause the newly created VFS
+** to become the default VFS.  The common case is for the fifth parameter
+** to be true.
+**
+** The call to vfstrace_register() simply creates the shim VFS that does
+** tracing.  The application must also arrange to use the new VFS for
+** all database connections that are created and for which tracing is 
+** desired.  This can be done by specifying the trace VFS using URI filename
+** notation, or by specifying the trace VFS as the 4th parameter to
+** sqlite3_open_v2() or by making the trace VFS be the default (by setting
+** the 5th parameter of vfstrace_register() to 1).
+**
+**
+** ENABLING VFSTRACE IN A COMMAND-LINE SHELL
+**
+** The SQLite command line shell implemented by the shell.c source file
+** can be used with this module.  To compile in -vfstrace support, first
+** gather this file (test_vfstrace.c), the shell source file (shell.c),
+** and the SQLite amalgamation source files (sqlite3.c, sqlite3.h) into
+** the working directory.  Then compile using a command like the following:
+**
+**    gcc -o sqlite3 -Os -I. -DSQLITE_ENABLE_VFSTRACE \
+**        -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \
+**        -DHAVE_READLINE -DHAVE_USLEEP=1 \
+**        shell.c test_vfstrace.c sqlite3.c -ldl -lreadline -lncurses
+**
+** The gcc command above works on Linux and provides (in addition to the
+** -vfstrace option) support for FTS3 and FTS4, RTREE, and command-line
+** editing using the readline library.  The command-line shell does not
+** use threads so we added -DSQLITE_THREADSAFE=0 just to make the code
+** run a little faster.   For compiling on a Mac, you'll probably need
+** to omit the -DHAVE_READLINE, the -lreadline, and the -lncurses options.
+** The compilation could be simplified to just this:
+**
+**    gcc -DSQLITE_ENABLE_VFSTRACE \
+**         shell.c test_vfstrace.c sqlite3.c -ldl -lpthread
+**
+** In this second example, all unnecessary options have been removed
+** Note that since the code is now threadsafe, we had to add the -lpthread
+** option to pull in the pthreads library.
+**
+** To cross-compile for windows using MinGW, a command like this might
+** work:
+**
+**    /opt/mingw/bin/i386-mingw32msvc-gcc -o sqlite3.exe -Os -I \
+**         -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_VFSTRACE \
+**         shell.c test_vfstrace.c sqlite3.c
+**
+** Similar compiler commands will work on different systems.  The key
+** invariants are (1) you must have -DSQLITE_ENABLE_VFSTRACE so that
+** the shell.c source file will know to include the -vfstrace command-line
+** option and (2) you must compile and link the three source files
+** shell,c, test_vfstrace.c, and sqlite3.c.  
+*/
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite3.h"
+
+/*
+** An instance of this structure is attached to the each trace VFS to
+** provide auxiliary information.
+*/
+typedef struct vfstrace_info vfstrace_info;
+struct vfstrace_info {
+  sqlite3_vfs *pRootVfs;              /* The underlying real VFS */
+  int (*xOut)(const char*, void*);    /* Send output here */
+  void *pOutArg;                      /* First argument to xOut */
+  const char *zVfsName;               /* Name of this trace-VFS */
+  sqlite3_vfs *pTraceVfs;             /* Pointer back to the trace VFS */
+};
+
+/*
+** The sqlite3_file object for the trace VFS
+*/
+typedef struct vfstrace_file vfstrace_file;
+struct vfstrace_file {
+  sqlite3_file base;        /* Base class.  Must be first */
+  vfstrace_info *pInfo;     /* The trace-VFS to which this file belongs */
+  const char *zFName;       /* Base name of the file */
+  sqlite3_file *pReal;      /* The real underlying file */
+};
+
+/*
+** Method declarations for vfstrace_file.
+*/
+static int vfstraceClose(sqlite3_file*);
+static int vfstraceRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int vfstraceWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64);
+static int vfstraceTruncate(sqlite3_file*, sqlite3_int64 size);
+static int vfstraceSync(sqlite3_file*, int flags);
+static int vfstraceFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int vfstraceLock(sqlite3_file*, int);
+static int vfstraceUnlock(sqlite3_file*, int);
+static int vfstraceCheckReservedLock(sqlite3_file*, int *);
+static int vfstraceFileControl(sqlite3_file*, int op, void *pArg);
+static int vfstraceSectorSize(sqlite3_file*);
+static int vfstraceDeviceCharacteristics(sqlite3_file*);
+static int vfstraceShmLock(sqlite3_file*,int,int,int);
+static int vfstraceShmMap(sqlite3_file*,int,int,int, void volatile **);
+static void vfstraceShmBarrier(sqlite3_file*);
+static int vfstraceShmUnmap(sqlite3_file*,int);
+
+/*
+** Method declarations for vfstrace_vfs.
+*/
+static int vfstraceOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int vfstraceDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int vfstraceAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int vfstraceFullPathname(sqlite3_vfs*, const char *zName, int, char *);
+static void *vfstraceDlOpen(sqlite3_vfs*, const char *zFilename);
+static void vfstraceDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
+static void (*vfstraceDlSym(sqlite3_vfs*,void*, const char *zSymbol))(void);
+static void vfstraceDlClose(sqlite3_vfs*, void*);
+static int vfstraceRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int vfstraceSleep(sqlite3_vfs*, int microseconds);
+static int vfstraceCurrentTime(sqlite3_vfs*, double*);
+static int vfstraceGetLastError(sqlite3_vfs*, int, char*);
+static int vfstraceCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int vfstraceSetSystemCall(sqlite3_vfs*,const char*, sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr vfstraceGetSystemCall(sqlite3_vfs*, const char *);
+static const char *vfstraceNextSystemCall(sqlite3_vfs*, const char *zName);
+
+/*
+** Return a pointer to the tail of the pathname.  Examples:
+**
+**     /home/drh/xyzzy.txt -> xyzzy.txt
+**     xyzzy.txt           -> xyzzy.txt
+*/
+static const char *fileTail(const char *z){
+  size_t i;
+  if( z==0 ) return 0;
+  i = strlen(z)-1;
+  while( i>0 && z[i-1]!='/' ){ i--; }
+  return &z[i];
+}
+
+/*
+** Send trace output defined by zFormat and subsequent arguments.
+*/
+static void vfstrace_printf(
+  vfstrace_info *pInfo,
+  const char *zFormat,
+  ...
+){
+  va_list ap;
+  char *zMsg;
+  va_start(ap, zFormat);
+  zMsg = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+  pInfo->xOut(zMsg, pInfo->pOutArg);
+  sqlite3_free(zMsg);
+}
+
+/*
+** Try to convert an error code into a symbolic name for that error code.
+*/
+static const char *vfstrace_errcode_name(int rc ){
+  const char *zVal = 0;
+  switch( rc ){
+    case SQLITE_OK:                 zVal = "SQLITE_OK";                 break;
+    case SQLITE_INTERNAL:           zVal = "SQLITE_INTERNAL";           break;
+    case SQLITE_ERROR:              zVal = "SQLITE_ERROR";              break;
+    case SQLITE_PERM:               zVal = "SQLITE_PERM";               break;
+    case SQLITE_ABORT:              zVal = "SQLITE_ABORT";              break;
+    case SQLITE_BUSY:               zVal = "SQLITE_BUSY";               break;
+    case SQLITE_LOCKED:             zVal = "SQLITE_LOCKED";             break;
+    case SQLITE_NOMEM:              zVal = "SQLITE_NOMEM";              break;
+    case SQLITE_READONLY:           zVal = "SQLITE_READONLY";           break;
+    case SQLITE_INTERRUPT:          zVal = "SQLITE_INTERRUPT";          break;
+    case SQLITE_IOERR:              zVal = "SQLITE_IOERR";              break;
+    case SQLITE_CORRUPT:            zVal = "SQLITE_CORRUPT";            break;
+    case SQLITE_NOTFOUND:           zVal = "SQLITE_NOTFOUND";           break;
+    case SQLITE_FULL:               zVal = "SQLITE_FULL";               break;
+    case SQLITE_CANTOPEN:           zVal = "SQLITE_CANTOPEN";           break;
+    case SQLITE_PROTOCOL:           zVal = "SQLITE_PROTOCOL";           break;
+    case SQLITE_EMPTY:              zVal = "SQLITE_EMPTY";              break;
+    case SQLITE_SCHEMA:             zVal = "SQLITE_SCHEMA";             break;
+    case SQLITE_TOOBIG:             zVal = "SQLITE_TOOBIG";             break;
+    case SQLITE_CONSTRAINT:         zVal = "SQLITE_CONSTRAINT";         break;
+    case SQLITE_MISMATCH:           zVal = "SQLITE_MISMATCH";           break;
+    case SQLITE_MISUSE:             zVal = "SQLITE_MISUSE";             break;
+    case SQLITE_NOLFS:              zVal = "SQLITE_NOLFS";              break;
+    case SQLITE_IOERR_READ:         zVal = "SQLITE_IOERR_READ";         break;
+    case SQLITE_IOERR_SHORT_READ:   zVal = "SQLITE_IOERR_SHORT_READ";   break;
+    case SQLITE_IOERR_WRITE:        zVal = "SQLITE_IOERR_WRITE";        break;
+    case SQLITE_IOERR_FSYNC:        zVal = "SQLITE_IOERR_FSYNC";        break;
+    case SQLITE_IOERR_DIR_FSYNC:    zVal = "SQLITE_IOERR_DIR_FSYNC";    break;
+    case SQLITE_IOERR_TRUNCATE:     zVal = "SQLITE_IOERR_TRUNCATE";     break;
+    case SQLITE_IOERR_FSTAT:        zVal = "SQLITE_IOERR_FSTAT";        break;
+    case SQLITE_IOERR_UNLOCK:       zVal = "SQLITE_IOERR_UNLOCK";       break;
+    case SQLITE_IOERR_RDLOCK:       zVal = "SQLITE_IOERR_RDLOCK";       break;
+    case SQLITE_IOERR_DELETE:       zVal = "SQLITE_IOERR_DELETE";       break;
+    case SQLITE_IOERR_BLOCKED:      zVal = "SQLITE_IOERR_BLOCKED";      break;
+    case SQLITE_IOERR_NOMEM:        zVal = "SQLITE_IOERR_NOMEM";        break;
+    case SQLITE_IOERR_ACCESS:       zVal = "SQLITE_IOERR_ACCESS";       break;
+    case SQLITE_IOERR_CHECKRESERVEDLOCK:
+                               zVal = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break;
+    case SQLITE_IOERR_LOCK:         zVal = "SQLITE_IOERR_LOCK";         break;
+    case SQLITE_IOERR_CLOSE:        zVal = "SQLITE_IOERR_CLOSE";        break;
+    case SQLITE_IOERR_DIR_CLOSE:    zVal = "SQLITE_IOERR_DIR_CLOSE";    break;
+    case SQLITE_IOERR_SHMOPEN:      zVal = "SQLITE_IOERR_SHMOPEN";      break;
+    case SQLITE_IOERR_SHMSIZE:      zVal = "SQLITE_IOERR_SHMSIZE";      break;
+    case SQLITE_IOERR_SHMLOCK:      zVal = "SQLITE_IOERR_SHMLOCK";      break;
+    case SQLITE_IOERR_SHMMAP:       zVal = "SQLITE_IOERR_SHMMAP";       break;
+    case SQLITE_IOERR_SEEK:         zVal = "SQLITE_IOERR_SEEK";         break;
+    case SQLITE_IOERR_GETTEMPPATH:  zVal = "SQLITE_IOERR_GETTEMPPATH";  break;
+    case SQLITE_IOERR_CONVPATH:     zVal = "SQLITE_IOERR_CONVPATH";     break;
+    case SQLITE_READONLY_DBMOVED:   zVal = "SQLITE_READONLY_DBMOVED";   break;
+    case SQLITE_LOCKED_SHAREDCACHE: zVal = "SQLITE_LOCKED_SHAREDCACHE"; break;
+    case SQLITE_BUSY_RECOVERY:      zVal = "SQLITE_BUSY_RECOVERY";      break;
+    case SQLITE_CANTOPEN_NOTEMPDIR: zVal = "SQLITE_CANTOPEN_NOTEMPDIR"; break;
+  }
+  return zVal;
+}
+
+/*
+** Convert value rc into a string and print it using zFormat.  zFormat
+** should have exactly one %s
+*/
+static void vfstrace_print_errcode(
+  vfstrace_info *pInfo,
+  const char *zFormat,
+  int rc
+){
+  const char *zVal;
+  char zBuf[50];
+  zVal = vfstrace_errcode_name(rc);
+  if( zVal==0 ){
+    zVal = vfstrace_errcode_name(rc&0xff);
+    if( zVal ){
+      sqlite3_snprintf(sizeof(zBuf), zBuf, "%s | 0x%x", zVal, rc&0xffff00);
+    }else{
+      sqlite3_snprintf(sizeof(zBuf), zBuf, "%d (0x%x)", rc, rc);
+    }
+    zVal = zBuf;
+  }
+  vfstrace_printf(pInfo, zFormat, zVal);
+}
+
+/*
+** Append to a buffer.
+*/
+static void strappend(char *z, int *pI, const char *zAppend){
+  int i = *pI;
+  while( zAppend[0] ){ z[i++] = *(zAppend++); }
+  z[i] = 0;
+  *pI = i;
+}
+
+/*
+** Close an vfstrace-file.
+*/
+static int vfstraceClose(sqlite3_file *pFile){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xClose(%s)", pInfo->zVfsName, p->zFName);
+  rc = p->pReal->pMethods->xClose(p->pReal);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  if( rc==SQLITE_OK ){
+    sqlite3_free((void*)p->base.pMethods);
+    p->base.pMethods = 0;
+  }
+  return rc;
+}
+
+/*
+** Read data from an vfstrace-file.
+*/
+static int vfstraceRead(
+  sqlite3_file *pFile, 
+  void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xRead(%s,n=%d,ofst=%lld)",
+                  pInfo->zVfsName, p->zFName, iAmt, iOfst);
+  rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  return rc;
+}
+
+/*
+** Write data to an vfstrace-file.
+*/
+static int vfstraceWrite(
+  sqlite3_file *pFile, 
+  const void *zBuf, 
+  int iAmt, 
+  sqlite_int64 iOfst
+){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xWrite(%s,n=%d,ofst=%lld)",
+                  pInfo->zVfsName, p->zFName, iAmt, iOfst);
+  rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  return rc;
+}
+
+/*
+** Truncate an vfstrace-file.
+*/
+static int vfstraceTruncate(sqlite3_file *pFile, sqlite_int64 size){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xTruncate(%s,%lld)", pInfo->zVfsName, p->zFName,
+                  size);
+  rc = p->pReal->pMethods->xTruncate(p->pReal, size);
+  vfstrace_printf(pInfo, " -> %d\n", rc);
+  return rc;
+}
+
+/*
+** Sync an vfstrace-file.
+*/
+static int vfstraceSync(sqlite3_file *pFile, int flags){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  int i;
+  char zBuf[100];
+  memcpy(zBuf, "|0", 3);
+  i = 0;
+  if( flags & SQLITE_SYNC_FULL )        strappend(zBuf, &i, "|FULL");
+  else if( flags & SQLITE_SYNC_NORMAL ) strappend(zBuf, &i, "|NORMAL");
+  if( flags & SQLITE_SYNC_DATAONLY )    strappend(zBuf, &i, "|DATAONLY");
+  if( flags & ~(SQLITE_SYNC_FULL|SQLITE_SYNC_DATAONLY) ){
+    sqlite3_snprintf(sizeof(zBuf)-i, &zBuf[i], "|0x%x", flags);
+  }
+  vfstrace_printf(pInfo, "%s.xSync(%s,%s)", pInfo->zVfsName, p->zFName,
+                  &zBuf[1]);
+  rc = p->pReal->pMethods->xSync(p->pReal, flags);
+  vfstrace_printf(pInfo, " -> %d\n", rc);
+  return rc;
+}
+
+/*
+** Return the current file-size of an vfstrace-file.
+*/
+static int vfstraceFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xFileSize(%s)", pInfo->zVfsName, p->zFName);
+  rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
+  vfstrace_print_errcode(pInfo, " -> %s,", rc);
+  vfstrace_printf(pInfo, " size=%lld\n", *pSize);
+  return rc;
+}
+
+/*
+** Return the name of a lock.
+*/
+static const char *lockName(int eLock){
+  const char *azLockNames[] = {
+     "NONE", "SHARED", "RESERVED", "PENDING", "EXCLUSIVE"
+  };
+  if( eLock<0 || eLock>=(int)(sizeof(azLockNames)/sizeof(azLockNames[0])) ){
+    return "???";
+  }else{
+    return azLockNames[eLock];
+  }
+}
+
+/*
+** Lock an vfstrace-file.
+*/
+static int vfstraceLock(sqlite3_file *pFile, int eLock){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xLock(%s,%s)", pInfo->zVfsName, p->zFName,
+                  lockName(eLock));
+  rc = p->pReal->pMethods->xLock(p->pReal, eLock);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  return rc;
+}
+
+/*
+** Unlock an vfstrace-file.
+*/
+static int vfstraceUnlock(sqlite3_file *pFile, int eLock){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xUnlock(%s,%s)", pInfo->zVfsName, p->zFName,
+                  lockName(eLock));
+  rc = p->pReal->pMethods->xUnlock(p->pReal, eLock);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  return rc;
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an vfstrace-file.
+*/
+static int vfstraceCheckReservedLock(sqlite3_file *pFile, int *pResOut){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xCheckReservedLock(%s,%d)", 
+                  pInfo->zVfsName, p->zFName);
+  rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
+  vfstrace_print_errcode(pInfo, " -> %s", rc);
+  vfstrace_printf(pInfo, ", out=%d\n", *pResOut);
+  return rc;
+}
+
+/*
+** File control method. For custom operations on an vfstrace-file.
+*/
+static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  char zBuf[100];
+  char zBuf2[100];
+  char *zOp;
+  char *zRVal = 0;
+  switch( op ){
+    case SQLITE_FCNTL_LOCKSTATE:           zOp = "LOCKSTATE";           break;
+    case SQLITE_GET_LOCKPROXYFILE:         zOp = "GET_LOCKPROXYFILE";   break;
+    case SQLITE_SET_LOCKPROXYFILE:         zOp = "SET_LOCKPROXYFILE";   break;
+    case SQLITE_LAST_ERRNO:                zOp = "LAST_ERRNO";          break;
+    case SQLITE_FCNTL_SIZE_HINT: {
+      sqlite3_snprintf(sizeof(zBuf), zBuf, "SIZE_HINT,%lld",
+                       *(sqlite3_int64*)pArg);
+      zOp = zBuf;
+      break;
+    }
+    case SQLITE_FCNTL_CHUNK_SIZE: {
+      sqlite3_snprintf(sizeof(zBuf), zBuf, "CHUNK_SIZE,%d", *(int*)pArg);
+      zOp = zBuf;
+      break;
+    }
+    case SQLITE_FCNTL_FILE_POINTER:        zOp = "FILE_POINTER";        break;
+    case SQLITE_FCNTL_WIN32_AV_RETRY:      zOp = "WIN32_AV_RETRY";      break;
+    case SQLITE_FCNTL_PERSIST_WAL: {
+       sqlite3_snprintf(sizeof(zBuf), zBuf, "PERSIST_WAL,%d", *(int*)pArg);
+       zOp = zBuf;
+       break;
+    }
+    case SQLITE_FCNTL_OVERWRITE:           zOp = "OVERWRITE";           break;
+    case SQLITE_FCNTL_VFSNAME:             zOp = "VFSNAME";             break;
+    case SQLITE_FCNTL_POWERSAFE_OVERWRITE: zOp = "POWERSAFE_OVERWRITE"; break;
+    case SQLITE_FCNTL_PRAGMA: {
+      const char *const* a = (const char*const*)pArg;
+      sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[1],a[2]);
+      zOp = zBuf;
+      break;
+    }
+    case SQLITE_FCNTL_BUSYHANDLER:         zOp = "BUSYHANDLER";         break;
+    case SQLITE_FCNTL_TEMPFILENAME:        zOp = "TEMPFILENAME";        break;
+    case SQLITE_FCNTL_MMAP_SIZE: {
+      sqlite3_int64 iMMap = *(sqlite3_int64*)pArg;
+      sqlite3_snprintf(sizeof(zBuf), zBuf, "MMAP_SIZE,%lld",iMMap);
+      zOp = zBuf;
+      break;
+    }
+    case SQLITE_FCNTL_TRACE:               zOp = "TRACE";               break;
+    case SQLITE_FCNTL_HAS_MOVED:           zOp = "HAS_MOVED";           break;
+    case SQLITE_FCNTL_SYNC:                zOp = "SYNC";                break;
+    case SQLITE_FCNTL_COMMIT_PHASETWO:     zOp = "COMMIT_PHASETWO";     break;
+    case SQLITE_FCNTL_WIN32_SET_HANDLE:    zOp = "WIN32_SET_HANDLE";    break;
+    case SQLITE_FCNTL_WAL_BLOCK:           zOp = "WAL_BLOCK";           break;
+    case SQLITE_FCNTL_ZIPVFS:              zOp = "ZIPVFS";              break; 
+    case SQLITE_FCNTL_RBU:                 zOp = "RBU";                 break;
+    case SQLITE_FCNTL_VFS_POINTER:         zOp = "VFS_POINTER";         break;
+    case SQLITE_FCNTL_JOURNAL_POINTER:     zOp = "JOURNAL_POINTER";     break;
+    case SQLITE_FCNTL_WIN32_GET_HANDLE:    zOp = "WIN32_GET_HANDLE";    break;
+    case SQLITE_FCNTL_PDB:                 zOp = "PDB";                 break;
+    case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE:  zOp = "BEGIN_ATOMIC_WRITE";  break;
+    case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: zOp = "COMMIT_ATOMIC_WRITE"; break;
+    case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: {
+       zOp = "ROLLBACK_ATOMIC_WRITE";
+       break;
+    }
+    case SQLITE_FCNTL_LOCK_TIMEOUT: {
+       sqlite3_snprintf(sizeof(zBuf), zBuf, "LOCK_TIMEOUT,%d", *(int*)pArg);
+       zOp = zBuf;
+       break;
+    }
+    case SQLITE_FCNTL_DATA_VERSION:        zOp = "DATA_VERSION";        break;
+    case SQLITE_FCNTL_SIZE_LIMIT:          zOp = "SIZE_LIMIT";          break;
+    case SQLITE_FCNTL_CKPT_DONE:           zOp = "CKPT_DONE";           break;
+    case SQLITE_FCNTL_RESERVE_BYTES:       zOp = "RESERVED_BYTES";      break;
+    case SQLITE_FCNTL_CKPT_START:          zOp = "CKPT_START";          break;
+    case SQLITE_FCNTL_EXTERNAL_READER:     zOp = "EXTERNAL_READER";     break;
+    case SQLITE_FCNTL_CKSM_FILE:           zOp = "CKSM_FILE";           break;
+    case SQLITE_FCNTL_RESET_CACHE:         zOp = "RESET_CACHE";         break;
+    case 0xca093fa0:                       zOp = "DB_UNCHANGED";        break;
+    default: {
+      sqlite3_snprintf(sizeof zBuf, zBuf, "%d", op);
+      zOp = zBuf;
+      break;
+    }
+  }
+  vfstrace_printf(pInfo, "%s.xFileControl(%s,%s)",
+                  pInfo->zVfsName, p->zFName, zOp);
+  rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg);
+  if( rc==SQLITE_OK ){
+    switch( op ){
+      case SQLITE_FCNTL_VFSNAME: {
+        *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z",
+                                    pInfo->zVfsName, *(char**)pArg);
+        zRVal = *(char**)pArg;
+        break;
+      }
+      case SQLITE_FCNTL_MMAP_SIZE: {
+        sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%lld", *(sqlite3_int64*)pArg);
+        zRVal = zBuf2;
+        break;
+      }
+      case SQLITE_FCNTL_HAS_MOVED:
+      case SQLITE_FCNTL_PERSIST_WAL: {
+        sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%d", *(int*)pArg);
+        zRVal = zBuf2;
+        break;
+      }
+      case SQLITE_FCNTL_PRAGMA:
+      case SQLITE_FCNTL_TEMPFILENAME: {
+        zRVal = *(char**)pArg;
+        break;
+      }
+    }
+  }
+  if( zRVal ){
+    vfstrace_print_errcode(pInfo, " -> %s", rc);
+    vfstrace_printf(pInfo, ", %s\n", zRVal);
+  }else{
+    vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  }
+  return rc;
+}
+
+/*
+** Return the sector-size in bytes for an vfstrace-file.
+*/
+static int vfstraceSectorSize(sqlite3_file *pFile){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xSectorSize(%s)", pInfo->zVfsName, p->zFName);
+  rc = p->pReal->pMethods->xSectorSize(p->pReal);
+  vfstrace_printf(pInfo, " -> %d\n", rc);
+  return rc;
+}
+
+/*
+** Return the device characteristic flags supported by an vfstrace-file.
+*/
+static int vfstraceDeviceCharacteristics(sqlite3_file *pFile){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xDeviceCharacteristics(%s)",
+                  pInfo->zVfsName, p->zFName);
+  rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
+  vfstrace_printf(pInfo, " -> 0x%08x\n", rc);
+  return rc;
+}
+
+/*
+** Shared-memory operations.
+*/
+static int vfstraceShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  char zLck[100];
+  int i = 0;
+  memcpy(zLck, "|0", 3);
+  if( flags & SQLITE_SHM_UNLOCK )    strappend(zLck, &i, "|UNLOCK");
+  if( flags & SQLITE_SHM_LOCK )      strappend(zLck, &i, "|LOCK");
+  if( flags & SQLITE_SHM_SHARED )    strappend(zLck, &i, "|SHARED");
+  if( flags & SQLITE_SHM_EXCLUSIVE ) strappend(zLck, &i, "|EXCLUSIVE");
+  if( flags & ~(0xf) ){
+     sqlite3_snprintf(sizeof(zLck)-i, &zLck[i], "|0x%x", flags);
+  }
+  vfstrace_printf(pInfo, "%s.xShmLock(%s,ofst=%d,n=%d,%s)",
+                  pInfo->zVfsName, p->zFName, ofst, n, &zLck[1]);
+  rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  return rc;
+}
+static int vfstraceShmMap(
+  sqlite3_file *pFile, 
+  int iRegion, 
+  int szRegion, 
+  int isWrite, 
+  void volatile **pp
+){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xShmMap(%s,iRegion=%d,szRegion=%d,isWrite=%d,*)",
+                  pInfo->zVfsName, p->zFName, iRegion, szRegion, isWrite);
+  rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  return rc;
+}
+static void vfstraceShmBarrier(sqlite3_file *pFile){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  vfstrace_printf(pInfo, "%s.xShmBarrier(%s)\n", pInfo->zVfsName, p->zFName);
+  p->pReal->pMethods->xShmBarrier(p->pReal);
+}
+static int vfstraceShmUnmap(sqlite3_file *pFile, int delFlag){
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = p->pInfo;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xShmUnmap(%s,delFlag=%d)",
+                  pInfo->zVfsName, p->zFName, delFlag);
+  rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  return rc;
+}
+
+
+
+/*
+** Open an vfstrace file handle.
+*/
+static int vfstraceOpen(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_file *pFile,
+  int flags,
+  int *pOutFlags
+){
+  int rc;
+  vfstrace_file *p = (vfstrace_file *)pFile;
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  p->pInfo = pInfo;
+  p->zFName = zName ? fileTail(zName) : "<temp>";
+  p->pReal = (sqlite3_file *)&p[1];
+  rc = pRoot->xOpen(pRoot, zName, p->pReal, flags, pOutFlags);
+  vfstrace_printf(pInfo, "%s.xOpen(%s,flags=0x%x)",
+                  pInfo->zVfsName, p->zFName, flags);
+  if( p->pReal->pMethods ){
+    sqlite3_io_methods *pNew = sqlite3_malloc( sizeof(*pNew) );
+    const sqlite3_io_methods *pSub = p->pReal->pMethods;
+    memset(pNew, 0, sizeof(*pNew));
+    pNew->iVersion = pSub->iVersion;
+    pNew->xClose = vfstraceClose;
+    pNew->xRead = vfstraceRead;
+    pNew->xWrite = vfstraceWrite;
+    pNew->xTruncate = vfstraceTruncate;
+    pNew->xSync = vfstraceSync;
+    pNew->xFileSize = vfstraceFileSize;
+    pNew->xLock = vfstraceLock;
+    pNew->xUnlock = vfstraceUnlock;
+    pNew->xCheckReservedLock = vfstraceCheckReservedLock;
+    pNew->xFileControl = vfstraceFileControl;
+    pNew->xSectorSize = vfstraceSectorSize;
+    pNew->xDeviceCharacteristics = vfstraceDeviceCharacteristics;
+    if( pNew->iVersion>=2 ){
+      pNew->xShmMap = pSub->xShmMap ? vfstraceShmMap : 0;
+      pNew->xShmLock = pSub->xShmLock ? vfstraceShmLock : 0;
+      pNew->xShmBarrier = pSub->xShmBarrier ? vfstraceShmBarrier : 0;
+      pNew->xShmUnmap = pSub->xShmUnmap ? vfstraceShmUnmap : 0;
+    }
+    pFile->pMethods = pNew;
+  }
+  vfstrace_print_errcode(pInfo, " -> %s", rc);
+  if( pOutFlags ){
+    vfstrace_printf(pInfo, ", outFlags=0x%x\n", *pOutFlags);
+  }else{
+    vfstrace_printf(pInfo, "\n");
+  }
+  return rc;
+}
+
+/*
+** Delete the file located at zPath. If the dirSync argument is true,
+** ensure the file-system modifications are synced to disk before
+** returning.
+*/
+static int vfstraceDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xDelete(\"%s\",%d)",
+                  pInfo->zVfsName, zPath, dirSync);
+  rc = pRoot->xDelete(pRoot, zPath, dirSync);
+  vfstrace_print_errcode(pInfo, " -> %s\n", rc);
+  return rc;
+}
+
+/*
+** Test for access permissions. Return true if the requested permission
+** is available, or false otherwise.
+*/
+static int vfstraceAccess(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int flags, 
+  int *pResOut
+){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xAccess(\"%s\",%d)",
+                  pInfo->zVfsName, zPath, flags);
+  rc = pRoot->xAccess(pRoot, zPath, flags, pResOut);
+  vfstrace_print_errcode(pInfo, " -> %s", rc);
+  vfstrace_printf(pInfo, ", out=%d\n", *pResOut);
+  return rc;
+}
+
+/*
+** Populate buffer zOut with the full canonical pathname corresponding
+** to the pathname in zPath. zOut is guaranteed to point to a buffer
+** of at least (DEVSYM_MAX_PATHNAME+1) bytes.
+*/
+static int vfstraceFullPathname(
+  sqlite3_vfs *pVfs, 
+  const char *zPath, 
+  int nOut, 
+  char *zOut
+){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  int rc;
+  vfstrace_printf(pInfo, "%s.xFullPathname(\"%s\")",
+                  pInfo->zVfsName, zPath);
+  rc = pRoot->xFullPathname(pRoot, zPath, nOut, zOut);
+  vfstrace_print_errcode(pInfo, " -> %s", rc);
+  vfstrace_printf(pInfo, ", out=\"%.*s\"\n", nOut, zOut);
+  return rc;
+}
+
+/*
+** Open the dynamic library located at zPath and return a handle.
+*/
+static void *vfstraceDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  vfstrace_printf(pInfo, "%s.xDlOpen(\"%s\")\n", pInfo->zVfsName, zPath);
+  return pRoot->xDlOpen(pRoot, zPath);
+}
+
+/*
+** Populate the buffer zErrMsg (size nByte bytes) with a human readable
+** utf-8 string describing the most recent error encountered associated 
+** with dynamic libraries.
+*/
+static void vfstraceDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  vfstrace_printf(pInfo, "%s.xDlError(%d)", pInfo->zVfsName, nByte);
+  pRoot->xDlError(pRoot, nByte, zErrMsg);
+  vfstrace_printf(pInfo, " -> \"%s\"", zErrMsg);
+}
+
+/*
+** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
+*/
+static void (*vfstraceDlSym(sqlite3_vfs *pVfs,void *p,const char *zSym))(void){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  vfstrace_printf(pInfo, "%s.xDlSym(\"%s\")\n", pInfo->zVfsName, zSym);
+  return pRoot->xDlSym(pRoot, p, zSym);
+}
+
+/*
+** Close the dynamic library handle pHandle.
+*/
+static void vfstraceDlClose(sqlite3_vfs *pVfs, void *pHandle){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  vfstrace_printf(pInfo, "%s.xDlOpen()\n", pInfo->zVfsName);
+  pRoot->xDlClose(pRoot, pHandle);
+}
+
+/*
+** Populate the buffer pointed to by zBufOut with nByte bytes of 
+** random data.
+*/
+static int vfstraceRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  vfstrace_printf(pInfo, "%s.xRandomness(%d)\n", pInfo->zVfsName, nByte);
+  return pRoot->xRandomness(pRoot, nByte, zBufOut);
+}
+
+/*
+** Sleep for nMicro microseconds. Return the number of microseconds 
+** actually slept.
+*/
+static int vfstraceSleep(sqlite3_vfs *pVfs, int nMicro){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  return pRoot->xSleep(pRoot, nMicro);
+}
+
+/*
+** Return the current time as a Julian Day number in *pTimeOut.
+*/
+static int vfstraceCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  return pRoot->xCurrentTime(pRoot, pTimeOut);
+}
+static int vfstraceCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  return pRoot->xCurrentTimeInt64(pRoot, pTimeOut);
+}
+
+/*
+** Return th3 most recent error code and message
+*/
+static int vfstraceGetLastError(sqlite3_vfs *pVfs, int iErr, char *zErr){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  return pRoot->xGetLastError(pRoot, iErr, zErr);
+}
+
+/*
+** Override system calls.
+*/
+static int vfstraceSetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName,
+  sqlite3_syscall_ptr pFunc
+){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  return pRoot->xSetSystemCall(pRoot, zName, pFunc);
+}
+static sqlite3_syscall_ptr vfstraceGetSystemCall(
+  sqlite3_vfs *pVfs,
+  const char *zName
+){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  return pRoot->xGetSystemCall(pRoot, zName);
+}
+static const char *vfstraceNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
+  vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
+  sqlite3_vfs *pRoot = pInfo->pRootVfs;
+  return pRoot->xNextSystemCall(pRoot, zName);
+}
+
+
+/*
+** Clients invoke this routine to construct a new trace-vfs shim.
+**
+** Return SQLITE_OK on success.  
+**
+** SQLITE_NOMEM is returned in the case of a memory allocation error.
+** SQLITE_NOTFOUND is returned if zOldVfsName does not exist.
+*/
+int vfstrace_register(
+   const char *zTraceName,           /* Name of the newly constructed VFS */
+   const char *zOldVfsName,          /* Name of the underlying VFS */
+   int (*xOut)(const char*,void*),   /* Output routine.  ex: fputs */
+   void *pOutArg,                    /* 2nd argument to xOut.  ex: stderr */
+   int makeDefault                   /* True to make the new VFS the default */
+){
+  sqlite3_vfs *pNew;
+  sqlite3_vfs *pRoot;
+  vfstrace_info *pInfo;
+  size_t nName;
+  size_t nByte;
+
+  pRoot = sqlite3_vfs_find(zOldVfsName);
+  if( pRoot==0 ) return SQLITE_NOTFOUND;
+  nName = strlen(zTraceName);
+  nByte = sizeof(*pNew) + sizeof(*pInfo) + nName + 1;
+  pNew = sqlite3_malloc64( nByte );
+  if( pNew==0 ) return SQLITE_NOMEM;
+  memset(pNew, 0, nByte);
+  pInfo = (vfstrace_info*)&pNew[1];
+  pNew->iVersion = pRoot->iVersion;
+  pNew->szOsFile = pRoot->szOsFile + sizeof(vfstrace_file);
+  pNew->mxPathname = pRoot->mxPathname;
+  pNew->zName = (char*)&pInfo[1];
+  memcpy((char*)&pInfo[1], zTraceName, nName+1);
+  pNew->pAppData = pInfo;
+  pNew->xOpen = vfstraceOpen;
+  pNew->xDelete = vfstraceDelete;
+  pNew->xAccess = vfstraceAccess;
+  pNew->xFullPathname = vfstraceFullPathname;
+  pNew->xDlOpen = pRoot->xDlOpen==0 ? 0 : vfstraceDlOpen;
+  pNew->xDlError = pRoot->xDlError==0 ? 0 : vfstraceDlError;
+  pNew->xDlSym = pRoot->xDlSym==0 ? 0 : vfstraceDlSym;
+  pNew->xDlClose = pRoot->xDlClose==0 ? 0 : vfstraceDlClose;
+  pNew->xRandomness = vfstraceRandomness;
+  pNew->xSleep = vfstraceSleep;
+  pNew->xCurrentTime = vfstraceCurrentTime;
+  pNew->xGetLastError = pRoot->xGetLastError==0 ? 0 : vfstraceGetLastError;
+  if( pNew->iVersion>=2 ){
+    pNew->xCurrentTimeInt64 = pRoot->xCurrentTimeInt64==0 ? 0 :
+                                   vfstraceCurrentTimeInt64;
+    if( pNew->iVersion>=3 ){
+      pNew->xSetSystemCall = pRoot->xSetSystemCall==0 ? 0 : 
+                                   vfstraceSetSystemCall;
+      pNew->xGetSystemCall = pRoot->xGetSystemCall==0 ? 0 : 
+                                   vfstraceGetSystemCall;
+      pNew->xNextSystemCall = pRoot->xNextSystemCall==0 ? 0 : 
+                                   vfstraceNextSystemCall;
+    }
+  }
+  pInfo->pRootVfs = pRoot;
+  pInfo->xOut = xOut;
+  pInfo->pOutArg = pOutArg;
+  pInfo->zVfsName = pNew->zName;
+  pInfo->pTraceVfs = pNew;
+  vfstrace_printf(pInfo, "%s.enabled_for(\"%s\")\n",
+       pInfo->zVfsName, pRoot->zName);
+  return sqlite3_vfs_register(pNew, makeDefault);
+}
+
+/*
+** Look for the named VFS.  If it is a TRACEVFS, then unregister it
+** and delete it.
+*/
+void vfstrace_unregister(const char *zTraceName){
+  sqlite3_vfs *pVfs = sqlite3_vfs_find(zTraceName);
+  if( pVfs==0 ) return;
+  if( pVfs->xOpen!=vfstraceOpen ) return;
+  sqlite3_vfs_unregister(pVfs);
+  sqlite3_free(pVfs);
+}
diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c
index feb7695d6..36688925d 100644
--- a/ext/rbu/sqlite3rbu.c
+++ b/ext/rbu/sqlite3rbu.c
@@ -336,6 +336,27 @@ struct RbuFrame {
   u32 iWalFrame;
 };
 
+#ifndef UNUSED_PARAMETER
+/*
+** The following macros are used to suppress compiler warnings and to
+** make it clear to human readers when a function parameter is deliberately
+** left unused within the body of a function. This usually happens when
+** a function is called via a function pointer. For example the
+** implementation of an SQL aggregate step callback may not use the
+** parameter indicating the number of arguments passed to the aggregate,
+** if it knows that this is enforced elsewhere.
+**
+** When a function parameter is not used at all within the body of a function,
+** it is generally named "NotUsed" or "NotUsed2" to make things even clearer.
+** However, these macros may also be used to suppress warnings related to
+** parameters that may or may not be used depending on compilation options.
+** For example those parameters only used in assert() statements. In these
+** cases the parameters are named as per the usual conventions.
+*/
+#define UNUSED_PARAMETER(x) (void)(x)
+#define UNUSED_PARAMETER2(x,y) UNUSED_PARAMETER(x),UNUSED_PARAMETER(y)
+#endif
+
 /*
 ** RBU handle.
 **
@@ -387,7 +408,7 @@ struct sqlite3rbu {
   int rc;                         /* Value returned by last rbu_step() call */
   char *zErrmsg;                  /* Error message if rc!=SQLITE_OK */
   int nStep;                      /* Rows processed for current object */
-  int nProgress;                  /* Rows processed for all objects */
+  sqlite3_int64 nProgress;        /* Rows processed for all objects */
   RbuObjIter objiter;             /* Iterator for skipping through tbl/idx */
   const char *zVfsName;           /* Name of automatically created rbu vfs */
   rbu_file *pTargetFd;            /* File handle open on target db */
@@ -504,7 +525,7 @@ static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){
      v = (v<<6) + c;
   }
   z--;
-  *pLen -= z - zStart;
+  *pLen -= (int)(z - zStart);
   *pz = (char*)z;
   return v;
 }
@@ -689,6 +710,7 @@ static void rbuFossilDeltaFunc(
   char *aOut;
 
   assert( argc==2 );
+  UNUSED_PARAMETER(argc);
 
   nOrig = sqlite3_value_bytes(argv[0]);
   aOrig = (const char*)sqlite3_value_blob(argv[0]);
@@ -2268,13 +2290,13 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){
           else if( c==')' ){
             nParen--;
             if( nParen==0 ){
-              int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan;
+              int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan);
               pIter->aIdxCol[iIdxCol++].nSpan = nSpan;
               i++;
               break;
             }
           }else if( c==',' && nParen==1 ){
-            int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan;
+            int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan);
             pIter->aIdxCol[iIdxCol++].nSpan = nSpan;
             pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1];
           }else if( c=='"' || c=='\'' || c=='`' ){
@@ -2964,6 +2986,8 @@ static void rbuFileSuffix3(const char *zBase, char *z){
     for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
     if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4);
   }
+#else
+  UNUSED_PARAMETER2(zBase,z);
 #endif
 }
 
@@ -3548,7 +3572,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){
           "(%d, %Q), "
           "(%d, %Q), "
           "(%d, %d), "
-          "(%d, %d), "
+          "(%d, %lld), "
           "(%d, %lld), "
           "(%d, %lld), "
           "(%d, %lld), "
@@ -3906,6 +3930,7 @@ static void rbuIndexCntFunc(
   sqlite3 *db = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain);
 
   assert( nVal==1 );
+  UNUSED_PARAMETER(nVal);
   
   rc = prepareFreeAndCollectError(db, &pStmt, &zErrmsg, 
       sqlite3_mprintf("SELECT count(*) FROM sqlite_schema "
@@ -4181,7 +4206,7 @@ sqlite3rbu *sqlite3rbu_vacuum(
 ){
   if( zTarget==0 ){ return rbuMisuseError(); }
   if( zState ){
-    int n = strlen(zState);
+    size_t n = strlen(zState);
     if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
       return rbuMisuseError();
     }
@@ -4398,6 +4423,7 @@ int sqlite3rbu_savestate(sqlite3rbu *p){
 */
 static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){
   int rc = SQLITE_OK;
+  UNUSED_PARAMETER(pArg);
 #if defined(_WIN32_WCE)
   {
     LPWSTR zWideOld;
@@ -5302,6 +5328,9 @@ static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
 ** No-op.
 */
 static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+  UNUSED_PARAMETER(pVfs);
+  UNUSED_PARAMETER(a);
+  UNUSED_PARAMETER(b);
   return 0;
 }
 
diff --git a/ext/session/sessionalter.test b/ext/session/sessionalter.test
index 34424cf27..bcff7c535 100644
--- a/ext/session/sessionalter.test
+++ b/ext/session/sessionalter.test
@@ -202,6 +202,7 @@ foreach {tn sql1 at sql2} {
   
     sqlite3changegroup grp
     grp schema db main
+    breakpoint
     grp add $C1
     grp add $C2
     set T1 [grp output]
diff --git a/ext/session/sessionfault3.test b/ext/session/sessionfault3.test
index af5a4cdb4..5af9c9ed7 100644
--- a/ext/session/sessionfault3.test
+++ b/ext/session/sessionfault3.test
@@ -56,4 +56,28 @@ do_faultsim_test 1 -faults oom* -prep {
   faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
 }
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(1, 'one');
+  INSERT INTO t1 VALUES(2, 'two');
+  ALTER TABLE t1 ADD COLUMN c DEFAULT 'abcdefghijklmnopqrstuvwxyz';
+}
+faultsim_save_and_close
+
+do_faultsim_test 2 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+  db eval {SELECT * FROM sqlite_schema}
+} -body {
+  sqlite3session S db main
+  S attach *
+  execsql {
+    DELETE FROM t1 WHERE a = 1;
+  }
+} -test {
+  faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
+  catch { S delete }
+}
+
 finish_test
diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c
index 7a8132bfa..5bab39b67 100644
--- a/ext/session/sqlite3session.c
+++ b/ext/session/sqlite3session.c
@@ -1757,16 +1757,19 @@ static void sessionPreupdateOneChange(
       for(i=0; i<(pTab->nCol-pTab->bRowid); i++){
         sqlite3_value *p = 0;
         if( op!=SQLITE_INSERT ){
-          TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p);
-          assert( trc==SQLITE_OK );
+          /* This may fail if the column has a non-NULL default and was added 
+          ** using ALTER TABLE ADD COLUMN after this record was created. */
+          rc = pSession->hook.xOld(pSession->hook.pCtx, i, &p);
         }else if( pTab->abPK[i] ){
           TESTONLY(int trc = ) pSession->hook.xNew(pSession->hook.pCtx, i, &p);
           assert( trc==SQLITE_OK );
         }
 
-        /* This may fail if SQLite value p contains a utf-16 string that must
-        ** be converted to utf-8 and an OOM error occurs while doing so. */
-        rc = sessionSerializeValue(0, p, &nByte);
+        if( rc==SQLITE_OK ){
+          /* This may fail if SQLite value p contains a utf-16 string that must
+          ** be converted to utf-8 and an OOM error occurs while doing so. */
+          rc = sessionSerializeValue(0, p, &nByte);
+        }
         if( rc!=SQLITE_OK ) goto error_out;
       }
       if( pTab->bRowid ){
@@ -5660,6 +5663,9 @@ static int sessionChangesetExtendRecord(
     sessionAppendBlob(pOut, aRec, nRec, &rc);
     if( rc==SQLITE_OK && pTab->pDfltStmt==0 ){
       rc = sessionPrepareDfltStmt(pGrp->db, pTab, &pTab->pDfltStmt);
+      if( rc==SQLITE_OK && SQLITE_ROW!=sqlite3_step(pTab->pDfltStmt) ){
+        rc = sqlite3_errcode(pGrp->db);
+      }
     }
     for(ii=nCol; rc==SQLITE_OK && ii<pTab->nCol; ii++){
       int eType = sqlite3_column_type(pTab->pDfltStmt, ii);
@@ -5676,6 +5682,7 @@ static int sessionChangesetExtendRecord(
           }
           if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){
             sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal);
+            pOut->nBuf += 8;
           }
           break;
         }
diff --git a/ext/session/test_session.c b/ext/session/test_session.c
index 79ad000e1..29aeadf53 100644
--- a/ext/session/test_session.c
+++ b/ext/session/test_session.c
@@ -383,10 +383,10 @@ static int SQLITE_TCLAPI test_session_cmd(
         { "rowid", SQLITE_SESSION_OBJCONFIG_ROWID },
         { 0, 0 }
       };
-      size_t sz = sizeof(aOpt[0]);
+      int sz = (int)sizeof(aOpt[0]);
 
       int iArg;
-      int iOpt;
+      Tcl_Size iOpt;
       if( Tcl_GetIndexFromObjStruct(interp,objv[2],aOpt,sz,"option",0,&iOpt) ){
         return TCL_ERROR;
       }
@@ -803,7 +803,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply(
   if( bV2 ){
     while( objc>1 ){
       const char *z1 = Tcl_GetString(objv[1]);
-      int n = strlen(z1);
+      int n = (int)strlen(z1);
       if( n>3 && n<=12 && 0==sqlite3_strnicmp("-nosavepoint", z1, n) ){
         flags |= SQLITE_CHANGESETAPPLY_NOSAVEPOINT;
       }
@@ -1119,7 +1119,7 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach(
 
   while( objc>1 ){
     char *zOpt = Tcl_GetString(objv[1]);
-    int nOpt = strlen(zOpt);
+    int nOpt = (int)strlen(zOpt);
     if( zOpt[0]!='-' ) break;
     if( nOpt<=7 && 0==sqlite3_strnicmp(zOpt, "-invert", nOpt) ){
       isInvert = 1;