Add experimental command "PRAGMA wal_blocking_checkpoint", which uses the busy-handler to block until all readers have finished in order to ensure the next writer will be able to wrap around to the start of the log file.

FossilOrigin-Name: 7e3fc2c833a5baa08820c499867b6902bdc2ed5a

10 files changed, 114 insertions, 39 deletions

diff --git a/src/btree.c b/src/btree.c
index 7e8c39feb..94100f48f 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -7935,8 +7935,15 @@ int sqlite3BtreeIsInTrans(Btree *p){
 **
 ** Return SQLITE_LOCKED if this or any other connection has an open 
 ** transaction on the shared-cache the argument Btree is connected to.
+**
+** If parameter bBlock is true, then the layers below invoke the 
+** busy-handler callback while waiting for readers to release locks so
+** that the entire WAL can be checkpointed. If it is false, then as
+** much as possible of the WAL is checkpointed without waiting for readers
+** to finish. bBlock is true for "PRAGMA wal_blocking_checkpoint" and false
+** for "PRAGMA wal_checkpoint".
 */
-int sqlite3BtreeCheckpoint(Btree *p){
+int sqlite3BtreeCheckpoint(Btree *p, int bBlock){
   int rc = SQLITE_OK;
   if( p ){
     BtShared *pBt = p->pBt;
@@ -7944,7 +7951,7 @@ int sqlite3BtreeCheckpoint(Btree *p){
     if( pBt->inTransaction!=TRANS_NONE ){
       rc = SQLITE_LOCKED;
     }else{
-      rc = sqlite3PagerCheckpoint(pBt->pPager);
+      rc = sqlite3PagerCheckpoint(pBt->pPager, bBlock);
     }
     sqlite3BtreeLeave(p);
   }
diff --git a/src/btree.h b/src/btree.h
index 39af03f96..ce86cdabb 100644
--- a/src/btree.h
+++ b/src/btree.h
@@ -207,7 +207,7 @@ void sqlite3BtreeCursorList(Btree*);
 #endif
 
 #ifndef SQLITE_OMIT_WAL
-  int sqlite3BtreeCheckpoint(Btree*);
+  int sqlite3BtreeCheckpoint(Btree*, int);
 #endif
 
 /*
diff --git a/src/main.c b/src/main.c
index 25216070d..7ce43aec5 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1361,7 +1361,7 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){
     rc = SQLITE_ERROR;
     sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb);
   }else{
-    rc = sqlite3Checkpoint(db, iDb);
+    rc = sqlite3Checkpoint(db, iDb, 0);
     sqlite3Error(db, rc, 0);
   }
   rc = sqlite3ApiExit(db, rc);
@@ -1387,8 +1387,11 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){
 ** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are
 ** checkpointed. If an error is encountered it is returned immediately -
 ** no attempt is made to checkpoint any remaining databases.
+**
+** Parameter bBlock is true for a blocking-checkpoint, false for an 
+** ordinary, non-blocking, checkpoint.
 */
-int sqlite3Checkpoint(sqlite3 *db, int iDb){
+int sqlite3Checkpoint(sqlite3 *db, int iDb, int bBlock){
   int rc = SQLITE_OK;             /* Return code */
   int i;                          /* Used to iterate through attached dbs */
 
@@ -1396,7 +1399,7 @@ int sqlite3Checkpoint(sqlite3 *db, int iDb){
 
   for(i=0; i<db->nDb && rc==SQLITE_OK; i++){
     if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){
-      rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt);
+      rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, bBlock);
     }
   }
 
diff --git a/src/pager.c b/src/pager.c
index b774af3a5..29f3bf1eb 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -6516,13 +6516,19 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){
 
 #ifndef SQLITE_OMIT_WAL
 /*
-** This function is called when the user invokes "PRAGMA checkpoint".
+** This function is called when the user invokes "PRAGMA wal_checkpoint",
+** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint()
+** or wal_blocking_checkpoint() API functions.
+**
+** Parameter bBlock is true for a blocking-checkpoint, false for an 
+** ordinary, non-blocking, checkpoint.
 */
-int sqlite3PagerCheckpoint(Pager *pPager){
+int sqlite3PagerCheckpoint(Pager *pPager, int bBlock){
   int rc = SQLITE_OK;
   if( pPager->pWal ){
     u8 *zBuf = (u8 *)pPager->pTmpSpace;
     rc = sqlite3WalCheckpoint(pPager->pWal,
+        (bBlock ? pPager->xBusyHandler : 0), pPager->pBusyHandlerArg,
         (pPager->noSync ? 0 : pPager->sync_flags),
         pPager->pageSize, zBuf
     );
diff --git a/src/pager.h b/src/pager.h
index c12afa7b8..e0396e99a 100644
--- a/src/pager.h
+++ b/src/pager.h
@@ -138,7 +138,7 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int n);
 int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);
 int sqlite3PagerSharedLock(Pager *pPager);
 
-int sqlite3PagerCheckpoint(Pager *pPager);
+int sqlite3PagerCheckpoint(Pager *pPager, int);
 int sqlite3PagerWalSupported(Pager *pPager);
 int sqlite3PagerWalCallback(Pager *pPager);
 int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen);
diff --git a/src/pragma.c b/src/pragma.c
index 362e77f29..7324e1bde 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1394,12 +1394,18 @@ void sqlite3Pragma(
 #ifndef SQLITE_OMIT_WAL
   /*
   **   PRAGMA [database.]wal_checkpoint
+  **   PRAGMA [database.]wal_blocking_checkpoint
   **
   ** Checkpoint the database.
   */
-  if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 ){
+  if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 
+   || sqlite3StrICmp(zLeft, "wal_blocking_checkpoint")==0 
+  ){
+    int bBlock = (zLeft[14]!=0);
+    int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED);
+    assert( bBlock==(sqlite3StrICmp(zLeft, "wal_checkpoint")!=0) );
     if( sqlite3ReadSchema(pParse) ) goto pragma_out;
-    sqlite3VdbeAddOp3(v, OP_Checkpoint, pId2->z?iDb:SQLITE_MAX_ATTACHED, 0, 0);
+    sqlite3VdbeAddOp2(v, OP_Checkpoint, iBt, bBlock);
   }else
 
   /*
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index c02a0e448..8b4fe599e 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3036,7 +3036,7 @@ CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
 int sqlite3TempInMemory(const sqlite3*);
 VTable *sqlite3GetVTable(sqlite3*, Table*);
 const char *sqlite3JournalModename(int);
-int sqlite3Checkpoint(sqlite3*, int);
+int sqlite3Checkpoint(sqlite3*, int, int);
 int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int);
 
 /* Declarations for functions in fkey.c. All of these are replaced by
diff --git a/src/vdbe.c b/src/vdbe.c
index 02d1a406c..dcff83f17 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -5216,13 +5216,13 @@ case OP_AggFinal: {
 }
 
 #ifndef SQLITE_OMIT_WAL
-/* Opcode: Checkpoint P1 * * * *
+/* Opcode: Checkpoint P1 P2 * * *
 **
 ** Checkpoint database P1. This is a no-op if P1 is not currently in
-** WAL mode.
+** WAL mode. If P2 is non-zero, this is a blocking checkpoint.
 */
 case OP_Checkpoint: {
-  rc = sqlite3Checkpoint(db, pOp->p1);
+  rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2);
   break;
 };  
 #endif
diff --git a/src/wal.c b/src/wal.c
index 3b217908b..5ff9f227e 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -1524,6 +1524,26 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){
 }
 
 /*
+** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and
+** n. If the attempt fails and parameter xBusy is not NULL, then it is a
+** busy-handler function. Invoke it and retry the lock until either the
+** lock is successfully obtained or the busy-handler returns 0.
+*/
+static int walBusyLock(
+  Wal *pWal,                      /* WAL connection */
+  int (*xBusy)(void*),            /* Function to call when busy */
+  void *pBusyArg,                 /* Context argument for xBusyHandler */
+  int lockIdx,                    /* Offset of first byte to lock */
+  int n                           /* Number of bytes to lock */
+){
+  int rc;
+  do {
+    rc = walLockExclusive(pWal, lockIdx, n);
+  }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) );
+  return rc;
+}
+
+/*
 ** Copy as much content as we can from the WAL back into the database file
 ** in response to an sqlite3_wal_checkpoint() request or the equivalent.
 **
@@ -1556,6 +1576,8 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){
 */
 static int walCheckpoint(
   Wal *pWal,                      /* Wal connection */
+  int (*xBusy)(void*),            /* Function to call when busy */
+  void *pBusyArg,                 /* Context argument for xBusyHandler */
   int sync_flags,                 /* Flags for OsSync() (or 0) */
   int nBuf,                       /* Size of zBuf in bytes */
   u8 *zBuf                        /* Temporary buffer to use */
@@ -1593,27 +1615,29 @@ static int walCheckpoint(
   ** overwrite database pages that are in use by active readers and thus
   ** cannot be backfilled from the WAL.
   */
-  mxSafeFrame = pWal->hdr.mxFrame;
-  mxPage = pWal->hdr.nPage;
-  pInfo = walCkptInfo(pWal);
-  for(i=1; i<WAL_NREADER; i++){
-    u32 y = pInfo->aReadMark[i];
-    if( mxSafeFrame>=y ){
-      assert( y<=pWal->hdr.mxFrame );
-      rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
-      if( rc==SQLITE_OK ){
-        pInfo->aReadMark[i] = READMARK_NOT_USED;
-        walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
-      }else if( rc==SQLITE_BUSY ){
-        mxSafeFrame = y;
-      }else{
-        goto walcheckpoint_out;
+  do {
+    mxSafeFrame = pWal->hdr.mxFrame;
+    mxPage = pWal->hdr.nPage;
+    pInfo = walCkptInfo(pWal);
+    for(i=1; i<WAL_NREADER; i++){
+      u32 y = pInfo->aReadMark[i];
+      if( mxSafeFrame>=y ){
+        assert( y<=pWal->hdr.mxFrame );
+        rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
+        if( rc==SQLITE_OK ){
+          pInfo->aReadMark[i] = READMARK_NOT_USED;
+          walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
+        }else if( rc==SQLITE_BUSY ){
+          mxSafeFrame = y;
+        }else{
+          goto walcheckpoint_out;
+        }
       }
     }
-  }
+  }while( xBusy && mxSafeFrame<pWal->hdr.mxFrame && xBusy(pBusyArg) );
 
   if( pInfo->nBackfill<mxSafeFrame
-   && (rc = walLockExclusive(pWal, WAL_READ_LOCK(0), 1))==SQLITE_OK
+   && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0), 1))==SQLITE_OK
   ){
     i64 nSize;                    /* Current size of database file */
     u32 nBackfill = pInfo->nBackfill;
@@ -1666,10 +1690,19 @@ static int walCheckpoint(
 
     /* Release the reader lock held while backfilling */
     walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
-  }else if( rc==SQLITE_BUSY ){
+
+    if( xBusy && rc==SQLITE_OK && pWal->hdr.mxFrame==mxSafeFrame ){
+      assert( pWal->writeLock );
+      rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
+      if( rc==SQLITE_OK ){
+        walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+      }
+    }
+  }
+
+  if( rc==SQLITE_BUSY ){
     /* Reset the return code so as not to report a checkpoint failure
-    ** just because active readers prevent any backfill.
-    */
+    ** just because there are active readers.  */
     rc = SQLITE_OK;
   }
 
@@ -1704,7 +1737,7 @@ int sqlite3WalClose(
       if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
         pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
       }
-      rc = sqlite3WalCheckpoint(pWal, sync_flags, nBuf, zBuf);
+      rc = sqlite3WalCheckpoint(pWal, 0, 0, sync_flags, nBuf, zBuf);
       if( rc==SQLITE_OK ){
         isDelete = 1;
       }
@@ -2619,9 +2652,14 @@ int sqlite3WalFrames(
 **
 ** Obtain a CHECKPOINT lock and then backfill as much information as
 ** we can from WAL into the database.
+**
+** If parameter xBusy is not NULL, it is a pointer to a busy-handler
+** callback. In this case this function runs a blocking checkpoint.
 */
 int sqlite3WalCheckpoint(
   Wal *pWal,                      /* Wal connection */
+  int (*xBusy)(void*),            /* Function to call when busy */
+  void *pBusyArg,                 /* Context argument for xBusyHandler */
   int sync_flags,                 /* Flags to sync db file with (or 0) */
   int nBuf,                       /* Size of temporary buffer */
   u8 *zBuf                        /* Temporary buffer to use */
@@ -2630,6 +2668,7 @@ int sqlite3WalCheckpoint(
   int isChanged = 0;              /* True if a new wal-index header is loaded */
 
   assert( pWal->ckptLock==0 );
+  assert( pWal->writeLock==0 );
 
   WALTRACE(("WAL%p: checkpoint begins\n", pWal));
   rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
@@ -2641,10 +2680,21 @@ int sqlite3WalCheckpoint(
   }
   pWal->ckptLock = 1;
 
+  /* If this is a blocking-checkpoint, then obtain the write-lock as well
+  ** to prevent any writers from running while the checkpoint is underway.
+  ** This has to be done before the call to walIndexReadHdr() below.
+  */
+  if( xBusy ){
+    rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1);
+    if( rc==SQLITE_OK ) pWal->writeLock = 1;
+  }
+
   /* Copy data from the log to the database file. */
-  rc = walIndexReadHdr(pWal, &isChanged);
   if( rc==SQLITE_OK ){
-    rc = walCheckpoint(pWal, sync_flags, nBuf, zBuf);
+    rc = walIndexReadHdr(pWal, &isChanged);
+  }
+  if( rc==SQLITE_OK ){
+    rc = walCheckpoint(pWal, xBusy, pBusyArg, sync_flags, nBuf, zBuf);
   }
   if( isChanged ){
     /* If a new wal-index header was loaded before the checkpoint was 
@@ -2657,6 +2707,7 @@ int sqlite3WalCheckpoint(
   }
 
   /* Release the locks. */
+  sqlite3WalEndWriteTransaction(pWal);
   walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
   pWal->ckptLock = 0;
   WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok"));
diff --git a/src/wal.h b/src/wal.h
index 35f695c88..e9c614fe8 100644
--- a/src/wal.h
+++ b/src/wal.h
@@ -32,7 +32,7 @@
 # define sqlite3WalSavepoint(y,z)
 # define sqlite3WalSavepointUndo(y,z)          0
 # define sqlite3WalFrames(u,v,w,x,y,z)         0
-# define sqlite3WalCheckpoint(u,v,w,x)         0
+# define sqlite3WalCheckpoint(u,v,w,x,y,z)     0
 # define sqlite3WalCallback(z)                 0
 # define sqlite3WalExclusiveMode(y,z)          0
 # define sqlite3WalHeapMemory(z)               0
@@ -86,6 +86,8 @@ int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int);
 /* Copy pages from the log to the database file */ 
 int sqlite3WalCheckpoint(
   Wal *pWal,                      /* Write-ahead log connection */
+  int (*xBusy)(void*),            /* Function to call when busy */
+  void *pBusyArg,                 /* Context argument for xBusyHandler */
   int sync_flags,                 /* Flags to sync db file with (or 0) */
   int nBuf,                       /* Size of buffer nBuf */
   u8 *zBuf                        /* Temporary buffer to use */