Add a %_config table to fts5.

FossilOrigin-Name: 83491c56661ca78f96020ba68184bb3fb19e674f

17 files changed, 287 insertions, 128 deletions

diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c
index fb3c0d719..b47b37aba 100644
--- a/ext/fts5/fts5.c
+++ b/ext/fts5/fts5.c
@@ -340,6 +340,11 @@ static int fts5InitVtab(
     rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
   }
 
+  /* Load the contents of %_config */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5ConfigLoad(pConfig);
+  }
+
   if( rc!=SQLITE_OK ){
     fts5FreeVtab(pTab, 0);
     pTab = 0;
@@ -887,7 +892,8 @@ static int fts5SeekCursor(Fts5Cursor *pCsr){
 ** This function is called to handle an FTS INSERT command. In other words,
 ** an INSERT statement of the form:
 **
-**     INSERT INTO fts(fts) VALUES($pVal)
+**     INSERT INTO fts(fts) VALUES($pCmd)
+**     INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
 **
 ** Argument pVal is the value assigned to column "fts" by the INSERT 
 ** statement. This function returns SQLITE_OK if successful, or an SQLite
@@ -897,28 +903,25 @@ static int fts5SeekCursor(Fts5Cursor *pCsr){
 ** INSERT Directives" section of the documentation. It should be updated if
 ** more commands are added to this function.
 */
-static int fts5SpecialCommand(Fts5Table *pTab, sqlite3_value *pVal){
-  const char *z = (const char*)sqlite3_value_text(pVal);
-  int n = sqlite3_value_bytes(pVal);
-  int rc = SQLITE_ERROR;
+static int fts5SpecialCommand(
+  Fts5Table *pTab,                /* Fts5 table object */
+  sqlite3_value *pCmd,            /* Value inserted into special column */
+  sqlite3_value *pVal             /* Value inserted into rowid column */
+){
+  const char *z = (const char*)sqlite3_value_text(pCmd);
+  int rc = SQLITE_OK;
+  int bError = 0;
 
   if( 0==sqlite3_stricmp("integrity-check", z) ){
     rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
-  }else
-
-  if( n>5 && 0==sqlite3_strnicmp("pgsz=", z, 5) ){
-    int pgsz = atoi(&z[5]);
-    if( pgsz<32 ) pgsz = 32;
-    sqlite3Fts5IndexPgsz(pTab->pIndex, pgsz);
-    rc = SQLITE_OK;
-  }else
-  
-  if( n>10 && 0==sqlite3_strnicmp("automerge=", z, 10) ){
-    int nAutomerge = atoi(&z[10]);
-    sqlite3Fts5IndexAutomerge(pTab->pIndex, nAutomerge);
-    rc = SQLITE_OK;
+  }else{
+    rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
+    if( rc==SQLITE_OK && bError ){
+      rc = SQLITE_ERROR;
+    }else{
+      rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal);
+    }
   }
-
   return rc;
 }
 
@@ -953,7 +956,9 @@ static int fts5UpdateMethod(
   assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
 
   if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
-    return fts5SpecialCommand(pTab, apVal[2 + pConfig->nCol]);
+    return fts5SpecialCommand(pTab, 
+        apVal[2 + pConfig->nCol], apVal[2 + pConfig->nCol + 1]
+    );
   }
 
   eType0 = sqlite3_value_type(apVal[0]);
@@ -1104,7 +1109,9 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){
         int *aInst;
         int iBest = -1;
         for(i=0; i<nIter; i++){
-          if( aIter[i].bEof==0 && (iBest<0 || aIter[i].iPos<iBest) ){
+          if( (aIter[i].bEof==0) 
+           && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
+          ){
             iBest = i;
           }
         }
diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 999777fcd..a29eb3f5d 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -58,6 +58,8 @@ typedef struct Fts5Config Fts5Config;
 /*
 ** An instance of the following structure encodes all information that can
 ** be gleaned from the CREATE VIRTUAL TABLE statement.
+**
+** And all information loaded from the %_config table.
 */
 struct Fts5Config {
   sqlite3 *db;                    /* Database handle */
@@ -69,6 +71,10 @@ struct Fts5Config {
   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   Fts5Tokenizer *pTok;
   fts5_tokenizer *pTokApi;
+
+  /* Values loaded from the %_config table */
+  int iCookie;                    /* Incremented when %_config is modified */
+  int pgsz;                       /* Approximate page size used in %_data */
 };
 
 int sqlite3Fts5ConfigParse(
@@ -87,6 +93,12 @@ int sqlite3Fts5Tokenize(
 
 void sqlite3Fts5Dequote(char *z);
 
+/* Load the contents of the %_config table */
+int sqlite3Fts5ConfigLoad(Fts5Config*);
+
+/* Set the value of a single config attribute */
+int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
+
 /*
 ** End of interface to code in fts5_config.c.
 **************************************************************************/
@@ -286,13 +298,6 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum);
 */
 int sqlite3Fts5IndexInit(sqlite3*);
 
-/*
-** Set the page size to use when writing. It doesn't matter if this
-** changes mid-transaction, or if inconsistent values are used by 
-** multiple clients.
-*/
-void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);
-
 void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMerge);
 
 /*
@@ -364,7 +369,7 @@ int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
 int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy);
 
 int sqlite3Fts5DropTable(Fts5Config*, const char *zPost);
-int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, char **pzErr);
+int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 
 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
 int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);
@@ -381,6 +386,8 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
 int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
 int sqlite3Fts5StorageRollback(Fts5Storage *p);
 
+int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);
+
 /*
 ** End of interface to code in fts5_storage.c.
 **************************************************************************/
diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index a039b7b53..aff871e9d 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -21,6 +21,7 @@ typedef struct HighlightContext HighlightContext;
 struct HighlightContext {
   const Fts5ExtensionApi *pApi;   /* API offered by current FTS version */
   Fts5Context *pFts;              /* First arg to pass to pApi functions */
+  int nInst;                      /* Total number of phrase instances */
   int iInst;                      /* Current phrase instance index */
   int iStart;                     /* First token of current phrase */
   int iEnd;                       /* Last token of current phrase */
@@ -35,57 +36,61 @@ struct HighlightContext {
   char *zOut;                     /* Output value */
 };
 
-static int fts5HighlightAppend(HighlightContext *p, const char *z, int n){
-  if( n<0 ) n = strlen(z);
-  p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
-  if( p->zOut==0 ) return SQLITE_NOMEM;
-  return SQLITE_OK;
+/*
+** Append text to the HighlightContext output string - p->zOut. Argument
+** z points to a buffer containing n bytes of text to append. If n is 
+** negative, everything up until the first '\0' is appended to the output.
+*/
+static void fts5HighlightAppend(
+  int *pRc, 
+  HighlightContext *p, 
+  const char *z, int n
+){
+  if( *pRc==SQLITE_OK ){
+    if( n<0 ) n = strlen(z);
+    p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
+    if( p->zOut==0 ) *pRc = SQLITE_NOMEM;
+  }
 }
 
 static int fts5HighlightCb(
   void *pContext,                 /* Pointer to HighlightContext object */
   const char *pToken,             /* Buffer containing token */
   int nToken,                     /* Size of token in bytes */
-  int iStart,                     /* Start offset of token */
-  int iEnd,                       /* End offset of token */
+  int iStartOff,                  /* Start offset of token */
+  int iEndOff,                    /* End offset of token */
   int iPos                        /* Position offset of token */
 ){
   HighlightContext *p = (HighlightContext*)pContext;
   int rc = SQLITE_OK;
 
   if( iPos==p->iStart ){
-    rc = fts5HighlightAppend(p, &p->zIn[p->iOff], iStart - p->iOff);
-    p->iOff = iStart;
-    if( rc==SQLITE_OK ){
-      rc = fts5HighlightAppend(p, p->zOpen, -1);
-    }
-  }
-  
-  if( rc==SQLITE_OK ){
-    rc = fts5HighlightAppend(p, &p->zIn[p->iOff], iEnd - p->iOff);
-    p->iOff = iEnd;
+    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
+    fts5HighlightAppend(&rc, p, p->zOpen, -1);
+    p->iOff = iStartOff;
   }
 
-  if( rc==SQLITE_OK && iPos==p->iEnd ){
+  if( iPos==p->iEnd ){
     int bClose = 1;
-    do{
+    for(p->iInst++; rc==SQLITE_OK && p->iInst<p->nInst; p->iInst++){
       int iP, iPCol, iOff;
-      rc = p->pApi->xInst(p->pFts, ++p->iInst, &iP, &iPCol, &iOff);
-      if( rc==SQLITE_RANGE || iPCol!=p->iCol ){
-        p->iStart = -1;
-        p->iEnd = -1;
-        rc = SQLITE_OK;
+      rc = p->pApi->xInst(p->pFts, p->iInst, &iP, &iPCol, &iOff);
+      if( iPCol!=p->iCol ){
+        p->iStart = p->iEnd = -1;
       }else{
-        iEnd = iOff - 1 + p->pApi->xPhraseSize(p->pFts, iP);
+        int iEnd = iOff - 1 + p->pApi->xPhraseSize(p->pFts, iP);
         if( iEnd<=p->iEnd ) continue;
         if( iOff<=p->iEnd ) bClose = 0;
         p->iStart = iOff;
         p->iEnd = iEnd;
       }
-    }while( 0 );
+      break;
+    }
 
-    if( rc==SQLITE_OK && bClose ){
-      rc = fts5HighlightAppend(p, p->zClose, -1);
+    if( bClose ){
+      fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
+      fts5HighlightAppend(&rc, p, p->zClose, -1);
+      p->iOff = iEndOff;
     }
   }
 
@@ -107,34 +112,33 @@ static void fts5HighlightFunction(
     sqlite3_result_error(pCtx, zErr, -1);
     return;
   }
+
   memset(&ctx, 0, sizeof(HighlightContext));
   ctx.iCol = sqlite3_value_int(apVal[0]);
   ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
-  rc = pApi->xColumnText(pFts, ctx.iCol, &ctx.zIn, &ctx.nIn);
   ctx.pApi = pApi;
   ctx.pFts = pFts;
+  rc = pApi->xColumnText(pFts, ctx.iCol, &ctx.zIn, &ctx.nIn);
+  if( rc==SQLITE_OK ) rc = pApi->xInstCount(pFts, &ctx.nInst);
 
   /* Find the first phrase instance in the right column. */
   ctx.iStart = -1;
   ctx.iEnd = -1;
-  while( rc==SQLITE_OK ){
+  for( ; ctx.iInst<ctx.nInst && rc==SQLITE_OK; ctx.iInst++){
     int iP, iPCol, iOff;
     rc = pApi->xInst(pFts, ctx.iInst, &iP, &iPCol, &iOff);
-    if( rc==SQLITE_OK && iPCol==ctx.iCol ){
+    if( iPCol==ctx.iCol ){
       ctx.iStart = iOff;
       ctx.iEnd = iOff - 1 + pApi->xPhraseSize(pFts, iP);
       break;
     }
-    ctx.iInst++;
   }
 
-  if( rc==SQLITE_OK || rc==SQLITE_RANGE ){
-    rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb);
-  }
   if( rc==SQLITE_OK ){
-    rc = fts5HighlightAppend(&ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
+    rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb);
   }
+  fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
 
   if( rc==SQLITE_OK ){
     sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index 68c340a48..98a6fe1af 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -15,6 +15,8 @@
 
 #include "fts5Int.h"
 
+#define FTS5_DEFAULT_PAGE_SIZE   1000
+
 /*
 ** Convert an SQL-style quoted string into a normal string by removing
 ** the quote characters.  The conversion is done in-place.  If the
@@ -295,4 +297,65 @@ int sqlite3Fts5Tokenize(
   return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
 }
 
+int sqlite3Fts5ConfigSetValue(
+  Fts5Config *pConfig, 
+  const char *zKey, 
+  sqlite3_value *pVal,
+  int *pbBadkey
+){
+  int rc = SQLITE_OK;
+  if(      0==sqlite3_stricmp(zKey, "cookie") ){
+    pConfig->iCookie = sqlite3_value_int(pVal);
+  }
+  else if( 0==sqlite3_stricmp(zKey, "pgsz") ){
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      pConfig->pgsz = sqlite3_value_int(pVal);
+    }else{
+      if( pbBadkey ) *pbBadkey = 1;
+    }
+  }
+  else if( 0==sqlite3_stricmp(zKey, "automerge") ){
+    // todo
+  }
+  else if( 0==sqlite3_stricmp(zKey, "rank") ){
+    // todo
+  }else{
+    if( pbBadkey ) *pbBadkey = 1;
+  }
+  return rc;
+}
+
+/*
+** Load the contents of the %_config table into memory.
+*/
+int sqlite3Fts5ConfigLoad(Fts5Config *pConfig){
+  const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
+  char *zSql;
+  sqlite3_stmt *p = 0;
+  int rc;
+
+  /* Set default values */
+  pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
+  pConfig->iCookie = 0;
+
+  zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
+    sqlite3_free(zSql);
+  }
+
+  assert( rc==SQLITE_OK || p==0 );
+  if( rc==SQLITE_OK ){
+    while( SQLITE_ROW==sqlite3_step(p) ){
+      const char *zK = (const char*)sqlite3_column_text(p, 0);
+      sqlite3_value *pVal = sqlite3_column_value(p, 1);
+      sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0);
+    }
+    rc = sqlite3_finalize(p);
+  }
+
+  return rc;
+}
 
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index b6923a3cf..262d5db97 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -41,8 +41,6 @@
 **
 */
 
-#define FTS5_DEFAULT_PAGE_SIZE   1000
-
 #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
 #define FTS5_MIN_MERGE       4    /* Minimum number of segments to merge */
 #define FTS5_CRISIS_MERGE   16    /* Maximum number of segments to merge */
@@ -290,7 +288,6 @@ typedef struct Fts5StructureSegment Fts5StructureSegment;
 struct Fts5Index {
   Fts5Config *pConfig;            /* Virtual table configuration */
   char *zDataTbl;                 /* Name of %_data table */
-  int pgsz;                       /* Target page size for this index */
   int nMinMerge;                  /* Minimum input segments in a merge */
   int nCrisisMerge;               /* Maximum allowed segments per level */
   int nWorkUnit;                  /* Leaf pages in a "unit" of work */
@@ -2535,12 +2532,15 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
 ** Discard all data currently cached in the hash-tables.
 */
 static void fts5IndexDiscardData(Fts5Index *p){
-  Fts5Config *pConfig = p->pConfig;
-  int i;
-  for(i=0; i<=pConfig->nPrefix; i++){
-    sqlite3Fts5HashClear(p->apHash[i]);
+  assert( p->apHash || p->nPendingData==0 );
+  if( p->apHash ){
+    Fts5Config *pConfig = p->pConfig;
+    int i;
+    for(i=0; i<=pConfig->nPrefix; i++){
+      sqlite3Fts5HashClear(p->apHash[i]);
+    }
+    p->nPendingData = 0;
   }
-  p->nPendingData = 0;
 }
 
 /*
@@ -2630,7 +2630,7 @@ static void fts5WriteBtreeTerm(
 
     fts5WriteBtreeNEmpty(p, pWriter);
 
-    if( pPage->buf.n>=p->pgsz ){
+    if( pPage->buf.n>=p->pConfig->pgsz ){
       /* pPage will be written to disk. The term will be written into the
       ** parent of pPage.  */
       i64 iRowid = FTS5_SEGMENT_ROWID(
@@ -2761,7 +2761,7 @@ static void fts5WriteAppendTerm(
   pWriter->bFirstRowidInDoclist = 1;
 
   /* If the current leaf page is full, flush it to disk. */
-  if( pPage->buf.n>=p->pgsz ){
+  if( pPage->buf.n>=p->pConfig->pgsz ){
     fts5WriteFlushLeaf(p, pWriter);
     pWriter->bFirstRowidInPage = 1;
   }
@@ -2796,7 +2796,7 @@ static void fts5WriteAppendRowid(
   pWriter->bFirstRowidInDoclist = 0;
   pWriter->bFirstRowidInPage = 0;
 
-  if( pPage->buf.n>=p->pgsz ){
+  if( pPage->buf.n>=p->pConfig->pgsz ){
     fts5WriteFlushLeaf(p, pWriter);
     pWriter->bFirstRowidInPage = 1;
   }
@@ -2809,7 +2809,7 @@ static void fts5WriteAppendPoslistInt(
 ){
   Fts5PageWriter *pPage = &pWriter->aWriter[0];
   fts5BufferAppendVarint(&p->rc, &pPage->buf, iVal);
-  if( pPage->buf.n>=p->pgsz ){
+  if( pPage->buf.n>=p->pConfig->pgsz ){
     fts5WriteFlushLeaf(p, pWriter);
     pWriter->bFirstRowidInPage = 1;
   }
@@ -2825,8 +2825,8 @@ static void fts5WriteAppendPoslistData(
   const u8 *a = aData;
   int n = nData;
   
-  while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pgsz ){
-    int nReq = p->pgsz - pPage->buf.n;
+  while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){
+    int nReq = p->pConfig->pgsz - pPage->buf.n;
     int nCopy = 0;
     while( nCopy<nReq ){
       i64 dummy;
@@ -3371,7 +3371,6 @@ int sqlite3Fts5IndexOpen(
 
   memset(p, 0, sizeof(Fts5Index));
   p->pConfig = pConfig;
-  p->pgsz = 1000;
   p->nMinMerge = FTS5_MIN_MERGE;
   p->nCrisisMerge = FTS5_CRISIS_MERGE;
   p->nWorkUnit = FTS5_WORK_UNIT;
@@ -3383,7 +3382,7 @@ int sqlite3Fts5IndexOpen(
     int i;
     Fts5Structure s;
     rc = sqlite3Fts5CreateTable(
-        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", pzErr
+        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
     );
     if( rc==SQLITE_OK ){
       memset(&s, 0, sizeof(Fts5Structure));
@@ -3987,13 +3986,6 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
 }
 
 /*
-** Set the target page size for the index object.
-*/
-void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz){
-  p->pgsz = pgsz;
-}
-
-/*
 ** Set the minimum number of segments that an auto-merge operation should
 ** attempt to merge together. A value of 1 sets the object to use the 
 ** compile time default. Zero or less disables auto-merge altogether.
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index ff0add5ba..bbe09874c 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -20,7 +20,7 @@ struct Fts5Storage {
   int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
   i64 nTotalRow;                  /* Total number of rows in FTS table */
   i64 *aTotalSize;                /* Total sizes of each column */ 
-  sqlite3_stmt *aStmt[9];
+  sqlite3_stmt *aStmt[10];
 };
 
 
@@ -43,6 +43,8 @@ struct Fts5Storage {
 
 #define FTS5_STMT_LOOKUP_DOCSIZE  8
 
+#define FTS5_STMT_REPLACE_CONFIG 9
+
 /*
 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
 ** Fts5Storage.pInsertDocsize - if they have not already been prepared.
@@ -70,6 +72,8 @@ static int fts5StorageGetStmt(
       "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
 
       "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
+
+      "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
     };
     Fts5Config *pConfig = p->pConfig;
     char *zSql = 0;
@@ -131,11 +135,13 @@ int sqlite3Fts5CreateTable(
   Fts5Config *pConfig,            /* FTS5 configuration */
   const char *zPost,              /* Shadow table to create (e.g. "content") */
   const char *zDefn,              /* Columns etc. for shadow table */
+  int bWithout,                   /* True for without rowid */
   char **pzErr                    /* OUT: Error message */
 ){
   int rc;
-  char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)",
-      pConfig->zDb, pConfig->zName, zPost, zDefn
+  char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)%s",
+      pConfig->zDb, pConfig->zName, zPost, zDefn, 
+      (bWithout ? " WITHOUT ROWID" :"")
   );
   if( zSql==0 ){
     rc = SQLITE_NOMEM;
@@ -193,12 +199,17 @@ int sqlite3Fts5StorageOpen(
       for(i=0; i<pConfig->nCol; i++){
         iOff += sprintf(&zDefn[iOff], ", c%d", i);
       }
-      rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, pzErr);
+      rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
     }
     sqlite3_free(zDefn);
     if( rc==SQLITE_OK ){
       rc = sqlite3Fts5CreateTable(
-          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", pzErr
+          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
+      );
+    }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3Fts5CreateTable(
+          pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
       );
     }
   }
@@ -225,7 +236,8 @@ int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy){
   /* If required, remove the shadow tables from the database */
   if( bDestroy ){
     rc = sqlite3Fts5DropTable(p->pConfig, "content");
-    if( rc==SQLITE_OK ) sqlite3Fts5DropTable(p->pConfig, "docsize");
+    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
+    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
   }
 
   sqlite3_free(p);
@@ -744,3 +756,20 @@ int sqlite3Fts5StorageRollback(Fts5Storage *p){
   return sqlite3Fts5IndexRollback(p->pIndex);
 }
 
+int sqlite3Fts5StorageConfigValue(
+  Fts5Storage *p, 
+  const char *z, 
+  sqlite3_value *pVal
+){
+  sqlite3_stmt *pReplace = 0;
+  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT);
+    sqlite3_bind_value(pReplace, 2, pVal);
+    sqlite3_step(pReplace);
+    rc = sqlite3_reset(pReplace);
+  }
+  return rc;
+}
+
+
diff --git a/manifest b/manifest
index 7d776a97d..2a109507b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\sauxiliary\shighlight()\sfunction\sto\sfts5.
-D 2014-11-24T16:24:33.456
+C Add\sa\s%_config\stable\sto\sfts5.
+D 2014-11-27T20:03:45.010
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -104,16 +104,16 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
 F ext/fts5/extract_api_docs.tcl 6320db4a1d0722a4e2069e661381ad75e9889786
-F ext/fts5/fts5.c d4b9895c5dc11c20493b3a9f09f4a0cdb0bc1438
+F ext/fts5/fts5.c 3c920d090b1cdbc69ba03acf7c9302a19be55cee
 F ext/fts5/fts5.h 72fc1e9995b1ddc254a487b9528614a83bd3dfb6
-F ext/fts5/fts5Int.h fd811979294410b10c1737392a9114510fc2a1be
-F ext/fts5/fts5_aux.c 2e467bdd93f23f049824411b326f77b9326cb61a
+F ext/fts5/fts5Int.h 63daceb6e421b9066e05c4e89651f27fa675be93
+F ext/fts5/fts5_aux.c 0e3e5fea6bf5772805afe14c95cb5f16e03e4b3f
 F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
-F ext/fts5/fts5_config.c a292fe73864086e51e7974d842cc09f6379fbae0
+F ext/fts5/fts5_config.c aae1470ca0e2125e758df5b612f26082a1dc254a
 F ext/fts5/fts5_expr.c d317be07d70223a6865444f17982570260b690a5
 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
-F ext/fts5/fts5_index.c 998c4aa0f003666afe85b6ff821476419ed245e9
-F ext/fts5/fts5_storage.c 5913aa01a1dada1c5e1a39e4cbb44e84c5f7f350
+F ext/fts5/fts5_index.c 5cb71b3922e50a23752fd6c11028acfe2f367850
+F ext/fts5/fts5_storage.c c28d1a88f45f83980eb32631c7421d7f5dd336fa
 F ext/fts5/fts5_tokenize.c 8360c0d1ae0d4696f3cc13f7c67a2db6011cdc5b
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
@@ -598,17 +598,18 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
 F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849
 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
-F test/fts5aa.test 16bf1dbb92d4d63c7c357b480b1a47309f654ad1
-F test/fts5ab.test 657d6dc5ddc57bfea4af1bb85204d4f3539cd3e8
-F test/fts5ac.test f38ceca8a43fa0ff86122bec72428a4067b17bc4
-F test/fts5ad.test d29ff407c70df470c9a8fcbfe5bc80efd662f2c4
-F test/fts5ae.test a514ee09be90723ccc9736edaef900a5af1c121a
+F test/fts5aa.test fb49e2db450f9bec900b05b6a85141695d6c2255
+F test/fts5ab.test 32ad48ca5317548dc6934585f6071b5e7ff03e61
+F test/fts5ac.test 3982268756a543cbf3ae508b6336f623136b754a
+F test/fts5ad.test 4e2e6a71fc7465eaaa32fd6ec318e657c6e7baa9
+F test/fts5ae.test 0c0712b1430158f976fce3564adf3e3713a4a93d
 F test/fts5af.test d24e3b0f879998ef5f60087272f8ab7b3a8fd4dc
 F test/fts5ag.test 1c6c188d1bdc41b2277db3f4ddfea7d90bf44ceb
-F test/fts5ah.test af9274cdb58a69780c7e57e61581990665ac0fb6
+F test/fts5ah.test c79b5107c2a47096f0e3473a4806ebc17f006cf4
 F test/fts5ai.test aa2b5fd0f8d2cf59ac0211111e63cbca3b40ed7d
-F test/fts5aj.test fe5c40216cac8072f29e454ee0540c7b89d17ccd
-F test/fts5ak.test 2c930afe32bd15b39a2c416fabe9fc7a36e3042e
+F test/fts5aj.test 947c957cdcfc8af7d428f8b82e82926b3b45a504
+F test/fts5ak.test e55bb0f3fac1291d32bc9485a3ee55a7d76f4d5f
+F test/fts5al.test 455b2bdc9f6ffb965a38a970a60c5075ee1e23bb
 F test/fts5ea.test afaf3497b43add578384dc1fd26b0342738abe87
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@@ -1205,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P fba0b5fc7eead07a4853e78e02d788e7c714f6cd
-R 0f9755027ecd37d1e64800649e8a07f5
+P 059092379f981eb919b500ce447006f9e645fc5a
+R 4ec91e9a7d6238c87465b8e2b7986d2c
 U dan
-Z 35a88e8d2ae30031588d0943e2aec6ce
+Z 1dea62df28b08fe11d603550a65329e7
diff --git a/manifest.uuid b/manifest.uuid
index 374484385..54ffb9092 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-059092379f981eb919b500ce447006f9e645fc5a
\ No newline at end of file
+83491c56661ca78f96020ba68184bb3fb19e674f
\ No newline at end of file
diff --git a/test/fts5aa.test b/test/fts5aa.test
index 412603468..9c715bfd4 100644
--- a/test/fts5aa.test
+++ b/test/fts5aa.test
@@ -30,6 +30,7 @@ do_execsql_test 1.0 {
   t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
   t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)}
   t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
+  t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID}
 }
 
 do_execsql_test 1.1 {
@@ -84,7 +85,7 @@ foreach {i x y} {
 reset_db
 do_execsql_test 4.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x,y);
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
 }
 foreach {i x y} {
    1  {g f d b f} {h h e i a}
@@ -108,7 +109,7 @@ foreach {i x y} {
 reset_db
 do_execsql_test 5.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x,y);
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
 }
 foreach {i x y} {
    1  {dd abc abc abc abcde} {aaa dd ddd ddd aab}
@@ -133,7 +134,7 @@ breakpoint
 reset_db
 do_execsql_test 6.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x,y);
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
 }
 
 do_execsql_test 6.1 {
@@ -151,7 +152,7 @@ reset_db
 expr srand(0)
 do_execsql_test 7.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
 }
 
 proc doc {} {
@@ -190,7 +191,7 @@ for {set i 1} {$i <= 10} {incr i} {
 reset_db
 do_execsql_test 8.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
 }
 
 do_execsql_test 8.1 {
@@ -207,7 +208,7 @@ expr srand(0)
 
 do_execsql_test 9.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3");
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
 }
 
 proc doc {} {
diff --git a/test/fts5ab.test b/test/fts5ab.test
index 4e2e9b13d..2fd3c047c 100644
--- a/test/fts5ab.test
+++ b/test/fts5ab.test
@@ -59,7 +59,7 @@ do_execsql_test 1.6 {
 reset_db
 do_execsql_test 2.1 {
   CREATE VIRTUAL TABLE t1 USING fts5(x);
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   INSERT INTO t1 VALUES('one');
   INSERT INTO t1 VALUES('two');
   INSERT INTO t1 VALUES('three');
@@ -99,7 +99,7 @@ foreach {tn expr res} {
 reset_db
 do_execsql_test 3.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a,b);
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
 }
 
 foreach {tn a b} {
diff --git a/test/fts5ac.test b/test/fts5ac.test
index be6177705..f3efa1410 100644
--- a/test/fts5ac.test
+++ b/test/fts5ac.test
@@ -25,7 +25,7 @@ ifcapable !fts5 {
 
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE xx USING fts5(x,y);
-  INSERT INTO xx(xx) VALUES('pgsz=32');
+  INSERT INTO xx(xx, rowid) VALUES('pgsz', 32);
 }
 
 set data {
diff --git a/test/fts5ad.test b/test/fts5ad.test
index 824444a86..8eabb7b97 100644
--- a/test/fts5ad.test
+++ b/test/fts5ad.test
@@ -55,12 +55,12 @@ foreach {tn match res} {
 foreach {T create} {
   2 {
     CREATE VIRTUAL TABLE t1 USING fts5(a, b);
-    INSERT INTO t1(t1) VALUES('pgsz=32');
+    INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   }
   
   3 {
     CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
-    INSERT INTO t1(t1) VALUES('pgsz=32');
+    INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
   }
 
 } {
diff --git a/test/fts5ae.test b/test/fts5ae.test
index b770d00c5..c29ec499e 100644
--- a/test/fts5ae.test
+++ b/test/fts5ae.test
@@ -25,7 +25,7 @@ ifcapable !fts5 {
 
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, b);
-  INSERT INTO t1(t1) VALUES('pgsz=32');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
 }
 
 do_execsql_test 1.1 {
diff --git a/test/fts5ah.test b/test/fts5ah.test
index fd78d2357..7ee6de973 100644
--- a/test/fts5ah.test
+++ b/test/fts5ah.test
@@ -28,7 +28,7 @@ ifcapable !fts5 {
 
 do_test 1.0 {
   execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
-  execsql { INSERT INTO t1(t1) VALUES('pgsz=128') }
+  execsql { INSERT INTO t1(t1, rowid) VALUES('pgsz', 128) }
   for {set i 1} {$i <= 10000} {incr i} {
     set v {x x x x x x x x x x x x x x x x x x x x}
     if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
diff --git a/test/fts5aj.test b/test/fts5aj.test
index 8b333ae56..6c8cd1827 100644
--- a/test/fts5aj.test
+++ b/test/fts5aj.test
@@ -46,7 +46,7 @@ proc structure {} {
 expr srand(0)
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x);
-  INSERT INTO t1(t1) VALUES('pgsz=64');
+  INSERT INTO t1(t1, rowid) VALUES('pgsz', 64);
 }
 
 for {set iTest 0} {$iTest < 50000} {incr iTest} {
diff --git a/test/fts5ak.test b/test/fts5ak.test
index 4d5b22b03..29d19bc4b 100644
--- a/test/fts5ak.test
+++ b/test/fts5ak.test
@@ -106,6 +106,18 @@ do_execsql_test 2.5 {
   {a [b c d e] f g h i j}
 }
 
+do_execsql_test 2.6.1 {
+  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d'
+} {
+  {a b c [d] e [f] g h i j}
+}
+
+do_execsql_test 2.6.2 {
+  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f'
+} {
+  {a b c [d] e [f] g h i j}
+}
+
 
 finish_test
 
diff --git a/test/fts5al.test b/test/fts5al.test
new file mode 100644
index 000000000..63e14a13a
--- /dev/null
+++ b/test/fts5al.test
@@ -0,0 +1,43 @@
+# 2014 November 24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+# Specifically, this function tests the %_config table.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix fts5al
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.1 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(x);
+  SELECT * FROM ft1_config;
+} {}
+
+do_execsql_test 1.2 {
+  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32);
+  SELECT * FROM ft1_config;
+} {pgsz 32}
+
+do_execsql_test 1.3 {
+  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
+  SELECT * FROM ft1_config;
+} {pgsz 64}
+
+finish_test
+