aboutsummaryrefslogtreecommitdiff
path: root/ext/fts5
diff options
context:
space:
mode:
Diffstat (limited to 'ext/fts5')
-rw-r--r--ext/fts5/fts5Int.h7
-rw-r--r--ext/fts5/fts5_config.c18
-rw-r--r--ext/fts5/fts5_expr.c167
-rw-r--r--ext/fts5/fts5_index.c108
-rw-r--r--ext/fts5/fts5_main.c6
-rw-r--r--ext/fts5/fts5_test_mi.c25
-rw-r--r--ext/fts5/fts5parse.y2
-rw-r--r--ext/fts5/test/fts5_common.tcl6
-rw-r--r--ext/fts5/test/fts5config.test16
-rw-r--r--ext/fts5/test/fts5eb.test12
-rw-r--r--ext/fts5/test/fts5fault8.test25
-rw-r--r--ext/fts5/test/fts5fuzz1.test93
-rw-r--r--ext/fts5/test/fts5merge.test68
-rw-r--r--ext/fts5/test/fts5optimize.test47
14 files changed, 501 insertions, 99 deletions
diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index af4041216..35f15abba 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -172,6 +172,7 @@ struct Fts5Config {
int pgsz; /* Approximate page size used in %_data */
int nAutomerge; /* 'automerge' setting */
int nCrisisMerge; /* Maximum allowed segments per level */
+ int nUsermerge; /* 'usermerge' setting */
int nHashSize; /* Bytes of memory for in-memory hash */
char *zRank; /* Name of rank function */
char *zRankArgs; /* Arguments to rank function */
@@ -700,6 +701,12 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
Fts5ExprNearset *pNear
);
+Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
+ Fts5Parse *pParse,
+ Fts5ExprNode *pLeft,
+ Fts5ExprNode *pRight
+);
+
Fts5ExprPhrase *sqlite3Fts5ParseTerm(
Fts5Parse *pParse,
Fts5ExprPhrase *pPhrase,
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index f49cede12..17fc43e01 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -18,6 +18,7 @@
#define FTS5_DEFAULT_PAGE_SIZE 4050
#define FTS5_DEFAULT_AUTOMERGE 4
+#define FTS5_DEFAULT_USERMERGE 4
#define FTS5_DEFAULT_CRISISMERGE 16
#define FTS5_DEFAULT_HASHSIZE (1024*1024)
@@ -441,7 +442,9 @@ static const char *fts5ConfigGobbleWord(
*pbQuoted = 1;
}else{
zRet = fts5ConfigSkipBareword(zIn);
- zOut[zRet-zIn] = '\0';
+ if( zRet ){
+ zOut[zRet-zIn] = '\0';
+ }
}
}
@@ -857,6 +860,18 @@ int sqlite3Fts5ConfigSetValue(
}
}
+ else if( 0==sqlite3_stricmp(zKey, "usermerge") ){
+ int nUsermerge = -1;
+ if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+ nUsermerge = sqlite3_value_int(pVal);
+ }
+ if( nUsermerge<2 || nUsermerge>16 ){
+ *pbBadkey = 1;
+ }else{
+ pConfig->nUsermerge = nUsermerge;
+ }
+ }
+
else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
int nCrisisMerge = -1;
if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
@@ -903,6 +918,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
/* Set default values */
pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
+ pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index d3f801b02..1e9be8117 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -258,6 +258,8 @@ int sqlite3Fts5ExprNew(
pNew->nPhrase = sParse.nPhrase;
sParse.apPhrase = 0;
}
+ }else{
+ sqlite3Fts5ParseNodeFree(sParse.pExpr);
}
sqlite3_free(sParse.apPhrase);
@@ -1268,6 +1270,8 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
if( Fts5NodeIsString(pNode) ){
/* Initialize all term iterators in the NEAR object. */
rc = fts5ExprNearInitAll(pExpr, pNode);
+ }else if( pNode->xNext==0 ){
+ pNode->bEof = 1;
}else{
int i;
int nEof = 0;
@@ -1319,23 +1323,22 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
*/
int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){
Fts5ExprNode *pRoot = p->pRoot;
- int rc = SQLITE_OK;
- if( pRoot->xNext ){
- p->pIndex = pIdx;
- p->bDesc = bDesc;
- rc = fts5ExprNodeFirst(p, pRoot);
+ int rc; /* Return code */
- /* If not at EOF but the current rowid occurs earlier than iFirst in
- ** the iteration order, move to document iFirst or later. */
- if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
- rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
- }
+ p->pIndex = pIdx;
+ p->bDesc = bDesc;
+ rc = fts5ExprNodeFirst(p, pRoot);
- /* If the iterator is not at a real match, skip forward until it is. */
- while( pRoot->bNomatch ){
- assert( pRoot->bEof==0 && rc==SQLITE_OK );
- rc = fts5ExprNodeNext(p, pRoot, 0, 0);
- }
+ /* If not at EOF but the current rowid occurs earlier than iFirst in
+ ** the iteration order, move to document iFirst or later. */
+ if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){
+ rc = fts5ExprNodeNext(p, pRoot, 1, iFirst);
+ }
+
+ /* If the iterator is not at a real match, skip forward until it is. */
+ while( pRoot->bNomatch ){
+ assert( pRoot->bEof==0 && rc==SQLITE_OK );
+ rc = fts5ExprNodeNext(p, pRoot, 0, 0);
}
return rc;
}
@@ -1444,6 +1447,21 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
sqlite3Fts5ParseNearsetFree(pNear);
sqlite3Fts5ParsePhraseFree(pPhrase);
}else{
+ if( pRet->nPhrase>0 ){
+ Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
+ assert( pLast==pParse->apPhrase[pParse->nPhrase-2] );
+ if( pPhrase->nTerm==0 ){
+ fts5ExprPhraseFree(pPhrase);
+ pRet->nPhrase--;
+ pParse->nPhrase--;
+ pPhrase = pLast;
+ }else if( pLast->nTerm==0 ){
+ fts5ExprPhraseFree(pLast);
+ pParse->apPhrase[pParse->nPhrase-2] = pPhrase;
+ pParse->nPhrase--;
+ pRet->nPhrase--;
+ }
+ }
pRet->apPhrase[pRet->nPhrase++] = pPhrase;
}
return pRet;
@@ -1476,8 +1494,7 @@ static int fts5ParseTokenize(
/* If an error has already occurred, this is a no-op */
if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
- assert( pPhrase==0 || pPhrase->nTerm>0 );
- if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
+ if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){
Fts5ExprTerm *pSyn;
int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
@@ -1578,7 +1595,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
pParse->rc = rc;
fts5ExprPhraseFree(sCtx.pPhrase);
sCtx.pPhrase = 0;
- }else if( sCtx.pPhrase ){
+ }else{
if( pAppend==0 ){
if( (pParse->nPhrase % 8)==0 ){
@@ -1595,9 +1612,14 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
pParse->nPhrase++;
}
+ if( sCtx.pPhrase==0 ){
+ /* This happens when parsing a token or quoted phrase that contains
+ ** no token characters at all. (e.g ... MATCH '""'). */
+ sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
+ }else if( sCtx.pPhrase->nTerm ){
+ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
+ }
pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
- assert( sCtx.pPhrase->nTerm>0 );
- sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
}
return sCtx.pPhrase;
@@ -1693,23 +1715,25 @@ void sqlite3Fts5ParseSetDistance(
Fts5ExprNearset *pNear,
Fts5Token *p
){
- int nNear = 0;
- int i;
- if( p->n ){
- for(i=0; i<p->n; i++){
- char c = (char)p->p[i];
- if( c<'0' || c>'9' ){
- sqlite3Fts5ParseError(
- pParse, "expected integer, got \"%.*s\"", p->n, p->p
- );
- return;
+ if( pNear ){
+ int nNear = 0;
+ int i;
+ if( p->n ){
+ for(i=0; i<p->n; i++){
+ char c = (char)p->p[i];
+ if( c<'0' || c>'9' ){
+ sqlite3Fts5ParseError(
+ pParse, "expected integer, got \"%.*s\"", p->n, p->p
+ );
+ return;
+ }
+ nNear = nNear * 10 + (p->p[i] - '0');
}
- nNear = nNear * 10 + (p->p[i] - '0');
+ }else{
+ nNear = FTS5_DEFAULT_NEARDIST;
}
- }else{
- nNear = FTS5_DEFAULT_NEARDIST;
+ pNear->nNear = nNear;
}
- pNear->nNear = nNear;
}
/*
@@ -1896,10 +1920,14 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
int iPhrase;
for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
pNear->apPhrase[iPhrase]->pNode = pRet;
+ if( pNear->apPhrase[iPhrase]->nTerm==0 ){
+ pRet->xNext = 0;
+ pRet->eType = FTS5_EOF;
+ }
}
if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL
- && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm!=1)
+ && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
){
assert( pParse->rc==SQLITE_OK );
pParse->rc = SQLITE_ERROR;
@@ -1928,6 +1956,70 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
return pRet;
}
+Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
+ Fts5Parse *pParse, /* Parse context */
+ Fts5ExprNode *pLeft, /* Left hand child expression */
+ Fts5ExprNode *pRight /* Right hand child expression */
+){
+ Fts5ExprNode *pRet = 0;
+ Fts5ExprNode *pPrev;
+
+ if( pParse->rc ){
+ sqlite3Fts5ParseNodeFree(pLeft);
+ sqlite3Fts5ParseNodeFree(pRight);
+ }else{
+
+ assert( pLeft->eType==FTS5_STRING
+ || pLeft->eType==FTS5_TERM
+ || pLeft->eType==FTS5_EOF
+ || pLeft->eType==FTS5_AND
+ );
+ assert( pRight->eType==FTS5_STRING
+ || pRight->eType==FTS5_TERM
+ || pRight->eType==FTS5_EOF
+ );
+
+ if( pLeft->eType==FTS5_AND ){
+ pPrev = pLeft->apChild[pLeft->nChild-1];
+ }else{
+ pPrev = pLeft;
+ }
+ assert( pPrev->eType==FTS5_STRING
+ || pPrev->eType==FTS5_TERM
+ || pPrev->eType==FTS5_EOF
+ );
+
+ if( pRight->eType==FTS5_EOF ){
+ assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
+ sqlite3Fts5ParseNodeFree(pRight);
+ pRet = pLeft;
+ pParse->nPhrase--;
+ }
+ else if( pPrev->eType==FTS5_EOF ){
+ Fts5ExprPhrase **ap;
+
+ if( pPrev==pLeft ){
+ pRet = pRight;
+ }else{
+ pLeft->apChild[pLeft->nChild-1] = pRight;
+ pRet = pLeft;
+ }
+
+ ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase];
+ assert( ap[0]==pPrev->pNear->apPhrase[0] );
+ memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase);
+ pParse->nPhrase--;
+
+ sqlite3Fts5ParseNodeFree(pPrev);
+ }
+ else{
+ pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND, pLeft, pRight, 0);
+ }
+ }
+
+ return pRet;
+}
+
static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
int nByte = 0;
Fts5ExprTerm *p;
@@ -2062,6 +2154,9 @@ static char *fts5ExprPrintTcl(
static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
char *zRet = 0;
+ if( pExpr->eType==0 ){
+ return sqlite3_mprintf("\"\"");
+ }else
if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){
Fts5ExprNearset *pNear = pExpr->pNear;
int i;
@@ -2122,7 +2217,7 @@ static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
zRet = 0;
}else{
int e = pExpr->apChild[i]->eType;
- int b = (e!=FTS5_STRING && e!=FTS5_TERM);
+ int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF);
zRet = fts5PrintfAppend(zRet, "%s%s%z%s",
(i==0 ? "" : zOp),
(b?"(":""), z, (b?")":"")
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index ac97a7d75..323e6cefd 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4179,13 +4179,17 @@ static void fts5IndexMergeLevel(
/*
** Do up to nPg pages of automerge work on the index.
+**
+** Return true if any changes were actually made, or false otherwise.
*/
-static void fts5IndexMerge(
+static int fts5IndexMerge(
Fts5Index *p, /* FTS5 backend object */
Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */
- int nPg /* Pages of work to do */
+ int nPg, /* Pages of work to do */
+ int nMin /* Minimum number of segments to merge */
){
int nRem = nPg;
+ int bRet = 0;
Fts5Structure *pStruct = *ppStruct;
while( nRem>0 && p->rc==SQLITE_OK ){
int iLvl; /* To iterate through levels */
@@ -4216,17 +4220,17 @@ static void fts5IndexMerge(
}
#endif
- if( nBest<p->pConfig->nAutomerge
- && pStruct->aLevel[iBestLvl].nMerge==0
- ){
+ if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
break;
}
+ bRet = 1;
fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
fts5StructurePromote(p, iBestLvl+1, pStruct);
}
}
*ppStruct = pStruct;
+ return bRet;
}
/*
@@ -4254,7 +4258,7 @@ static void fts5IndexAutomerge(
pStruct->nWriteCounter += nLeaf;
nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel);
- fts5IndexMerge(p, ppStruct, nRem);
+ fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge);
}
}
@@ -4474,25 +4478,38 @@ static void fts5IndexFlush(Fts5Index *p){
}
}
-
-int sqlite3Fts5IndexOptimize(Fts5Index *p){
- Fts5Structure *pStruct;
+static Fts5Structure *fts5IndexOptimizeStruct(
+ Fts5Index *p,
+ Fts5Structure *pStruct
+){
Fts5Structure *pNew = 0;
- int nSeg = 0;
-
- assert( p->rc==SQLITE_OK );
- fts5IndexFlush(p);
- pStruct = fts5StructureRead(p);
+ int nByte = sizeof(Fts5Structure);
+ int nSeg = pStruct->nSegment;
+ int i;
- if( pStruct ){
- assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
- nSeg = pStruct->nSegment;
- if( nSeg>1 ){
- int nByte = sizeof(Fts5Structure);
- nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
- pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
+ /* Figure out if this structure requires optimization. A structure does
+ ** not require optimization if either:
+ **
+ ** + it consists of fewer than two segments, or
+ ** + all segments are on the same level, or
+ ** + all segments except one are currently inputs to a merge operation.
+ **
+ ** In the first case, return NULL. In the second, increment the ref-count
+ ** on *pStruct and return a copy of the pointer to it.
+ */
+ if( nSeg<2 ) return 0;
+ for(i=0; i<pStruct->nLevel; i++){
+ int nThis = pStruct->aLevel[i].nSeg;
+ if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){
+ fts5StructureRef(pStruct);
+ return pStruct;
}
+ assert( pStruct->aLevel[i].nMerge<=nThis );
}
+
+ nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
+ pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
+
if( pNew ){
Fts5StructureLevel *pLvl;
int nByte = nSeg * sizeof(Fts5StructureSegment);
@@ -4520,8 +4537,26 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
}
}
+ return pNew;
+}
+
+int sqlite3Fts5IndexOptimize(Fts5Index *p){
+ Fts5Structure *pStruct;
+ Fts5Structure *pNew = 0;
+
+ assert( p->rc==SQLITE_OK );
+ fts5IndexFlush(p);
+ pStruct = fts5StructureRead(p);
+
+ if( pStruct ){
+ pNew = fts5IndexOptimizeStruct(p, pStruct);
+ }
+ fts5StructureRelease(pStruct);
+
+ assert( pNew==0 || pNew->nSegment>0 );
if( pNew ){
- int iLvl = pNew->nLevel-1;
+ int iLvl;
+ for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){}
while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
int nRem = FTS5_OPT_WORK_UNIT;
fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
@@ -4531,20 +4566,31 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
fts5StructureRelease(pNew);
}
- fts5StructureRelease(pStruct);
return fts5IndexReturn(p);
}
+/*
+** This is called to implement the special "VALUES('merge', $nMerge)"
+** INSERT command.
+*/
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
- Fts5Structure *pStruct;
-
- pStruct = fts5StructureRead(p);
- if( pStruct && pStruct->nLevel ){
- fts5IndexMerge(p, &pStruct, nMerge);
- fts5StructureWrite(p, pStruct);
+ Fts5Structure *pStruct = fts5StructureRead(p);
+ if( pStruct ){
+ int nMin = p->pConfig->nUsermerge;
+ if( nMerge<0 ){
+ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
+ fts5StructureRelease(pStruct);
+ pStruct = pNew;
+ nMin = 2;
+ nMerge = nMerge*-1;
+ }
+ if( pStruct && pStruct->nLevel ){
+ if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){
+ fts5StructureWrite(p, pStruct);
+ }
+ }
+ fts5StructureRelease(pStruct);
}
- fts5StructureRelease(pStruct);
-
return fts5IndexReturn(p);
}
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 28f3f3e62..dcd131c74 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1511,13 +1511,13 @@ static int fts5UpdateMethod(
rc = SQLITE_ERROR;
}
- /* Case 1: DELETE */
+ /* DELETE */
else if( nArg==1 ){
i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */
rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
}
- /* Case 2: INSERT */
+ /* INSERT */
else if( eType0!=SQLITE_INTEGER ){
/* If this is a REPLACE, first remove the current entry (if any) */
if( eConflict==SQLITE_REPLACE
@@ -1529,7 +1529,7 @@ static int fts5UpdateMethod(
fts5StorageInsert(&rc, pTab, apVal, pRowid);
}
- /* Case 2: UPDATE */
+ /* UPDATE */
else{
i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */
i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */
diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c
index bc6d01f12..a905b85bb 100644
--- a/ext/fts5/fts5_test_mi.c
+++ b/ext/fts5/fts5_test_mi.c
@@ -68,18 +68,22 @@ struct Fts5MatchinfoCtx {
** If an error occurs, return NULL and leave an error in the database
** handle (accessible using sqlite3_errcode()/errmsg()).
*/
-static fts5_api *fts5_api_from_db(sqlite3 *db){
- fts5_api *pRet = 0;
+static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
sqlite3_stmt *pStmt = 0;
+ int rc;
- if( SQLITE_OK==sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0)
- && SQLITE_ROW==sqlite3_step(pStmt)
- && sizeof(pRet)==sqlite3_column_bytes(pStmt, 0)
- ){
- memcpy(&pRet, sqlite3_column_blob(pStmt, 0), sizeof(pRet));
+ *ppApi = 0;
+ rc = sqlite3_prepare(db, "SELECT fts5()", -1, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pStmt)
+ && sizeof(fts5_api*)==sqlite3_column_bytes(pStmt, 0)
+ ){
+ memcpy(ppApi, sqlite3_column_blob(pStmt, 0), sizeof(fts5_api*));
+ }
+ rc = sqlite3_finalize(pStmt);
}
- sqlite3_finalize(pStmt);
- return pRet;
+
+ return rc;
}
@@ -399,7 +403,8 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){
/* Extract the FTS5 API pointer from the database handle. The
** fts5_api_from_db() function above is copied verbatim from the
** FTS5 documentation. Refer there for details. */
- pApi = fts5_api_from_db(db);
+ rc = fts5_api_from_db(db, &pApi);
+ if( rc!=SQLITE_OK ) return rc;
/* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
** with this database handle, or an error (OOM perhaps?) has occurred.
diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y
index 2bdf4b09b..1607d3846 100644
--- a/ext/fts5/fts5parse.y
+++ b/ext/fts5/fts5parse.y
@@ -104,7 +104,7 @@ expr(A) ::= exprlist(X). {A = X;}
exprlist(A) ::= cnearset(X). {A = X;}
exprlist(A) ::= exprlist(X) cnearset(Y). {
- A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
+ A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
}
cnearset(A) ::= nearset(X). {
diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl
index 32691d1c8..0f371dcfd 100644
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -159,6 +159,12 @@ proc fts5_aux_test_functions {db} {
}
}
+proc fts5_segcount {tbl} {
+ set N 0
+ foreach n [fts5_level_segs $tbl] { incr N $n }
+ set N
+}
+
proc fts5_level_segs {tbl} {
set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
set ret [list]
diff --git a/ext/fts5/test/fts5config.test b/ext/fts5/test/fts5config.test
index c30a59724..386d112e7 100644
--- a/ext/fts5/test/fts5config.test
+++ b/ext/fts5/test/fts5config.test
@@ -247,5 +247,21 @@ do_catchsql_test 12.1 {
INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
} {1 {SQL logic error or missing database}}
+#-------------------------------------------------------------------------
+# errors in the 'usermerge' option
+#
+do_execsql_test 13.0 {
+ CREATE VIRTUAL TABLE tt USING fts5(ttt);
+}
+foreach {tn val} {
+ 1 -1
+ 2 4.2
+ 3 17
+ 4 1
+} {
+ set sql "INSERT INTO tt(tt, rank) VALUES('usermerge', $val)"
+ do_catchsql_test 13.$tn $sql {1 {SQL logic error or missing database}}
+}
+
finish_test
diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test
index 820539604..69418aae6 100644
--- a/ext/fts5/test/fts5eb.test
+++ b/ext/fts5/test/fts5eb.test
@@ -33,12 +33,12 @@ foreach {tn expr res} {
1 {abc} {"abc"}
2 {abc ""} {"abc"}
3 {""} {}
- 4 {abc OR ""} {"abc"}
- 5 {abc NOT ""} {"abc"}
- 6 {abc AND ""} {"abc"}
- 7 {"" OR abc} {"abc"}
- 8 {"" NOT abc} {"abc"}
- 9 {"" AND abc} {"abc"}
+ 4 {abc OR ""} {"abc" OR ""}
+ 5 {abc NOT ""} {"abc" NOT ""}
+ 6 {abc AND ""} {"abc" AND ""}
+ 7 {"" OR abc} {"" OR "abc"}
+ 8 {"" NOT abc} {"" NOT "abc"}
+ 9 {"" AND abc} {"" AND "abc"}
10 {abc + "" + def} {"abc" + "def"}
11 {abc "" def} {"abc" AND "def"}
12 {r+e OR w} {"r" + "e" OR "w"}
diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test
index ae5849495..c613490e5 100644
--- a/ext/fts5/test/fts5fault8.test
+++ b/ext/fts5/test/fts5fault8.test
@@ -54,7 +54,32 @@ foreach_detail_mode $testprefix {
faultsim_test_result {0 {1 3}} {1 SQLITE_NOMEM}
}
}
+
} ;# foreach_detail_mode...
+
+do_execsql_test 4.0 {
+ CREATE VIRTUAL TABLE x2 USING fts5(a);
+ INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
+ INSERT INTO x2(x2, rank) VALUES('pgsz', 32);
+ INSERT INTO x2 VALUES('a b c d');
+ INSERT INTO x2 VALUES('e f g h');
+ INSERT INTO x2 VALUES('i j k l');
+ INSERT INTO x2 VALUES('m n o p');
+ INSERT INTO x2 VALUES('q r s t');
+ INSERT INTO x2 VALUES('u v w x');
+ INSERT INTO x2 VALUES('y z a b');
+}
+faultsim_save_and_close
+
+do_faultsim_test 4 -faults oom-* -prep {
+ faultsim_restore_and_reopen
+} -body {
+ execsql { INSERT INTO x2(x2) VALUES('optimize') }
+} -test {
+ faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
+}
+
+
finish_test
diff --git a/ext/fts5/test/fts5fuzz1.test b/ext/fts5/test/fts5fuzz1.test
new file mode 100644
index 000000000..599d7bcc8
--- /dev/null
+++ b/ext/fts5/test/fts5fuzz1.test
@@ -0,0 +1,93 @@
+# 2014 June 17
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library. The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+return_if_no_fts5
+set testprefix fts5fuzz1
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_catchsql_test 1.1 {
+ CREATE VIRTUAL TABLE f1 USING fts5(a b);
+} {/1 {parse error in.*}/}
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.1 {
+ CREATE VIRTUAL TABLE f1 USING fts5(a, b);
+ INSERT INTO f1 VALUES('a b', 'c d');
+ INSERT INTO f1 VALUES('e f', 'a b');
+}
+
+do_execsql_test 2.2.1 {
+ SELECT rowid FROM f1('""');
+} {}
+
+do_execsql_test 2.2.2 {
+ SELECT rowid FROM f1('"" AND a');
+} {}
+
+
+do_execsql_test 2.2.3 {
+ SELECT rowid FROM f1('"" a');
+} {1 2}
+
+do_execsql_test 2.2.4 {
+ SELECT rowid FROM f1('"" OR a');
+} {1 2}
+
+do_execsql_test 2.3 {
+ SELECT a, b FROM f1('NEAR("")');
+} {}
+
+do_execsql_test 2.4 {
+ SELECT a, b FROM f1('NEAR("", 5)');
+} {}
+
+do_execsql_test 2.5 {
+ SELECT a, b FROM f1('NEAR("" c, 5)');
+} {{a b} {c d}}
+
+do_execsql_test 2.6 {
+ SELECT a, b FROM f1('NEAR("" c d, 5)');
+} {{a b} {c d}}
+
+do_execsql_test 2.7 {
+ SELECT a, b FROM f1('NEAR(c d, 5)');
+} {{a b} {c d}}
+
+do_execsql_test 2.8 {
+ SELECT rowid FROM f1('NEAR("a" "b", 5)');
+} {1 2}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 3.2 {
+ CREATE VIRTUAL TABLE f2 USING fts5(o, t, tokenize="ascii separators abc");
+ SELECT * FROM f2('a+4');
+} {}
+
+
+
+#-------------------------------------------------------------------------
+reset_db
+do_catchsql_test 4.1 {
+ CREATE VIRTUAL TABLE f2 USING fts5(o, t);
+ SELECT * FROM f2('(8 AND 9)`AND 10');
+} {1 {fts5: syntax error near "`"}}
+
+finish_test
+
diff --git a/ext/fts5/test/fts5merge.test b/ext/fts5/test/fts5merge.test
index 9dd1ecd02..73e006a7d 100644
--- a/ext/fts5/test/fts5merge.test
+++ b/ext/fts5/test/fts5merge.test
@@ -45,7 +45,7 @@ proc do_merge1_test {testname nRowPerSeg} {
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<$::nRowPerSeg)
INSERT INTO x8 SELECT repeat('x y ', i % 16) FROM ii;
- INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
}
for {set tn 1} {[lindex [fts5_level_segs x8] 0]>0} {incr tn} {
@@ -84,9 +84,9 @@ proc do_merge2_test {testname nRow} {
execsql { INSERT INTO x8 VALUES( rnddoc(($i%16) + 5) ) }
while {[not_merged x8]} {
execsql {
- INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
INSERT INTO x8(x8, rank) VALUES('merge', 1);
- INSERT INTO x8(x8, rank) VALUES('automerge', 16);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 16);
INSERT INTO x8(x8) VALUES('integrity-check');
}
}
@@ -104,9 +104,9 @@ do_merge2_test 2.2 10
do_merge2_test 2.3 20
#-------------------------------------------------------------------------
-# Test that an auto-merge will complete any merge that has already been
+# Test that a merge will complete any merge that has already been
# started, even if the number of input segments is less than the current
-# value of the 'automerge' configuration parameter.
+# value of the 'usermerge' configuration parameter.
#
db func rnddoc fts5_rnddoc
@@ -119,7 +119,7 @@ do_execsql_test 3.1 {
}
do_test 3.2 {
execsql {
- INSERT INTO x8(x8, rank) VALUES('automerge', 4);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 4);
INSERT INTO x8(x8, rank) VALUES('merge', 1);
}
fts5_level_segs x8
@@ -127,14 +127,14 @@ do_test 3.2 {
do_test 3.3 {
execsql {
- INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
INSERT INTO x8(x8, rank) VALUES('merge', 1);
}
fts5_level_segs x8
} {2 1}
do_test 3.4 {
- execsql { INSERT INTO x8(x8, rank) VALUES('automerge', 4) }
+ execsql { INSERT INTO x8(x8, rank) VALUES('usermerge', 4) }
while {[not_merged x8]} {
execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1) }
}
@@ -176,7 +176,7 @@ foreach {tn pgsz} {
INSERT INTO x8 SELECT mydoc() FROM ii;
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
INSERT INTO x8 SELECT mydoc() FROM ii;
- INSERT INTO x8(x8, rank) VALUES('automerge', 2);
+ INSERT INTO x8(x8, rank) VALUES('usermerge', 2);
}
set expect [mycount]
@@ -190,5 +190,55 @@ foreach {tn pgsz} {
# db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r }
}
+#-------------------------------------------------------------------------
+# Test that the 'merge' command does not modify the database if there is
+# no work to do.
+
+do_execsql_test 5.1 {
+ CREATE VIRTUAL TABLE x9 USING fts5(one, two);
+ INSERT INTO x9(x9, rank) VALUES('pgsz', 32);
+ INSERT INTO x9(x9, rank) VALUES('automerge', 2);
+ INSERT INTO x9(x9, rank) VALUES('usermerge', 2);
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+ INSERT INTO x9 VALUES(rnddoc(100), rnddoc(100));
+}
+
+do_test 5.2 {
+ while 1 {
+ set nChange [db total_changes]
+ execsql { INSERT INTO x9(x9, rank) VALUES('merge', 1); }
+ set nChange [expr [db total_changes] - $nChange]
+ #puts $nChange
+ if {$nChange<2} break
+ }
+} {}
+
+
+#--------------------------------------------------------------------------
+# Test that running 'merge' on an empty database does not cause a
+# problem.
+#
+reset_db
+do_execsql_test 6.0 {
+ CREATE VIRTUAL TABLE g1 USING fts5(a, b);
+}
+do_execsql_test 6.1 {
+ INSERT INTO g1(g1, rank) VALUES('merge', 10);
+}
+do_execsql_test 6.2 {
+ INSERT INTO g1(g1, rank) VALUES('merge', -10);
+}
+do_execsql_test 6.3 {
+ INSERT INTO g1(g1) VALUES('integrity-check');
+}
+
+
+
finish_test
diff --git a/ext/fts5/test/fts5optimize.test b/ext/fts5/test/fts5optimize.test
index 984af8c53..3ef6d8a16 100644
--- a/ext/fts5/test/fts5optimize.test
+++ b/ext/fts5/test/fts5optimize.test
@@ -20,6 +20,12 @@ ifcapable !fts5 {
return
}
+#
+# 1.* - Warm body tests for index optimization using ('optimize')
+#
+# 2.* - Warm body tests for index optimization using ('merge', -1)
+#
+
proc rnddoc {nWord} {
set vocab {a b c d e f g h i j k l m n o p q r s t u v w x y z}
set nVocab [llength $vocab]
@@ -30,14 +36,12 @@ proc rnddoc {nWord} {
return $ret
}
-
foreach {tn nStep} {
1 2
2 10
3 50
4 500
} {
-if {$tn!=4} continue
reset_db
db func rnddoc rnddoc
do_execsql_test 1.$tn.1 {
@@ -60,7 +64,46 @@ if {$tn!=4} continue
do_execsql_test 1.$tn.5 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
+
+ do_test 1.$tn.6 { fts5_segcount t1 } 1
}
+foreach {tn nStep} {
+ 1 2
+ 2 10
+ 3 50
+ 4 500
+} {
+ reset_db
+ db func rnddoc rnddoc
+ do_execsql_test 1.$tn.1 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+ }
+ do_test 2.$tn.2 {
+ for {set i 0} {$i < $nStep} {incr i} {
+ execsql { INSERT INTO t1 VALUES( rnddoc(5), rnddoc(5) ) }
+ }
+ } {}
+
+ do_execsql_test 2.$tn.3 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+ }
+
+ do_test 2.$tn.4 {
+ execsql { INSERT INTO t1(t1, rank) VALUES('merge', -1) }
+ while 1 {
+ set c [db total_changes]
+ execsql { INSERT INTO t1(t1, rank) VALUES('merge', 1) }
+ set c [expr [db total_changes]-$c]
+ if {$c<2} break
+ }
+ } {}
+
+ do_execsql_test 2.$tn.5 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+ }
+
+ do_test 2.$tn.6 { fts5_segcount t1 } 1
+}
finish_test
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-19 16:32:55 +0000