Merge the latest fixes and enhancements from trunk into the

coroutines-exp2 branch.

FossilOrigin-Name: 1c5f41986f5af181bf389675361c9f176e9195e847319f07ebd5c87992ded38b

10 files changed, 237 insertions, 139 deletions

diff --git a/src/memdb.c b/src/memdb.c
index 7485f51ff..4ea3cffde 100644
--- a/src/memdb.c
+++ b/src/memdb.c
@@ -370,21 +370,33 @@ static int memdbLock(sqlite3_file *pFile, int eLock){
   int rc = SQLITE_OK;
   if( eLock==pThis->eLock ) return SQLITE_OK;
   memdbEnter(p);
+  assert( p->nWrLock==0 || p->nWrLock==1 );  /* No more than 1 write lock */
   if( eLock>SQLITE_LOCK_SHARED ){
     assert( pThis->eLock>=SQLITE_LOCK_SHARED );
     if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){
       rc = SQLITE_READONLY;
     }else if( eLock==SQLITE_LOCK_EXCLUSIVE ){
-      /* Taking an EXCLUSIVE lock. Fail if we only have SHARED and any
-      ** other client has any kind of write-lock. Also fail if any other
-      ** client is holding read-lock. */
-      if( pThis->eLock<=SQLITE_LOCK_SHARED && p->nWrLock ){
-        rc = SQLITE_BUSY;
-      }else if( p->nRdLock>1 ){
+      /* We never go for an EXCLUSIVE lock unless we already hold SHARED or
+      ** higher */
+      assert( pThis->eLock>=SQLITE_LOCK_SHARED );
+      testcase( pThis->eLock==SQLITE_LOCK_SHARED );
+
+      /* Because we are holding SHARED or more, there must be at least
+      ** one read lock */
+      assert( p->nRdLock>0 );
+
+      /* The only way that there can be an existing write lock is if we
+      ** currently hold it.  Otherwise, we would have never been able to
+      ** promote from NONE to SHARED. */
+      assert( p->nWrLock==0 || pThis->eLock>SQLITE_LOCK_SHARED );
+
+      if( p->nRdLock>1 ){
+        /* Cannot take EXCLUSIVE if somebody else is holding SHARED */
         rc = SQLITE_BUSY;
+      }else{
+        p->nWrLock = 1;
       }
-      p->nWrLock = 1;
-    }else if( pThis->eLock<=SQLITE_LOCK_SHARED ){
+    }else if( ALWAYS(pThis->eLock<=SQLITE_LOCK_SHARED) ){
       /* Upgrading to RESERVED or PENDING from SHARED. Fail if any other
       ** client has a write-lock of any kind.  */
       if( p->nWrLock ){
diff --git a/src/shell.c.in b/src/shell.c.in
index 85e708551..e4b054f45 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -482,24 +482,26 @@ static char continuePrompt[PROMPT_LEN_MAX];
 */
 #ifdef SQLITE_OMIT_DYNAPROMPT
 # define CONTINUATION_PROMPT continuePrompt
-# define CONTINUE_PROMPT_RESET(p)
+# define CONTINUE_PROMPT_RESET
 # define CONTINUE_PROMPT_AWAITS(p,s)
 # define CONTINUE_PROMPT_AWAITC(p,c)
 # define CONTINUE_PAREN_INCR(p,n)
-# define CONTINUE_PROMPT_STATE 0
-# define SCAN_TRACKER_REFTYPE void*
+# define CONTINUE_PROMPT_PSTATE 0
+typedef void *t_NoDynaPrompt;
+# define SCAN_TRACKER_REFTYPE t_NoDynaPrompt
 #else
 # define CONTINUATION_PROMPT dynamicContinuePrompt()
-# define CONTINUE_PROMPT_RESET(p) \
-  if(p) (setLexemeOpen(p,0,0), trackParenLevel(p,0))
+# define CONTINUE_PROMPT_RESET \
+  do {setLexemeOpen(&dynPrompt,0,0); trackParenLevel(&dynPrompt,0);} while(0)
 # define CONTINUE_PROMPT_AWAITS(p,s) \
   if(p && stdin_is_interactive) setLexemeOpen(p, s, 0)
 # define CONTINUE_PROMPT_AWAITC(p,c) \
   if(p && stdin_is_interactive) setLexemeOpen(p, 0, c)
 # define CONTINUE_PAREN_INCR(p,n) \
   if(p && stdin_is_interactive) (trackParenLevel(p,n))
-# define SCAN_TRACKER_REFTYPE struct DynaPrompt *
-# define CONTINUE_PROMPT_STATE (pDynPrompt)
+# define CONTINUE_PROMPT_PSTATE (&dynPrompt)
+typedef struct DynaPrompt *t_DynaPromptRef;
+# define SCAN_TRACKER_REFTYPE t_DynaPromptRef
 
 static struct DynaPrompt {
   char dynamicPrompt[PROMPT_LEN_MAX];
@@ -507,7 +509,6 @@ static struct DynaPrompt {
   int inParenLevel;
   char *zScannerAwaits;
 } dynPrompt = { {0}, {0}, 0, 0 };
-static struct DynaPrompt *pDynPrompt = &dynPrompt;
 
 /* Record parenthesis nesting level change, or force level to 0. */
 static void trackParenLevel(struct DynaPrompt *p, int ni){
@@ -536,17 +537,17 @@ static char *dynamicContinuePrompt(void){
     if( dynPrompt.zScannerAwaits ){
       int ncp = strlen(continuePrompt), ndp = strlen(dynPrompt.zScannerAwaits);
       if( ndp > ncp-3 ) return continuePrompt;
-      strncpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits, ndp);
+      strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits);
       while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' ';
       strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3,
 	      PROMPT_LEN_MAX-4);
     }else{
       if( dynPrompt.inParenLevel>9 ){
-	strncpy(dynPrompt.dynamicPrompt, "(..", 3);
+	strncpy(dynPrompt.dynamicPrompt, "(..", 4);
       }else if( dynPrompt.inParenLevel<0 ){
-	strncpy(dynPrompt.dynamicPrompt, ")x!", 3);
+	strncpy(dynPrompt.dynamicPrompt, ")x!", 4);
       }else{
-	strncpy(dynPrompt.dynamicPrompt, "(x.", 3);
+	strncpy(dynPrompt.dynamicPrompt, "(x.", 4);
 	dynPrompt.dynamicPrompt[2] = (char)('0'+dynPrompt.inParenLevel);
       }
       strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3, PROMPT_LEN_MAX-4);
@@ -10261,12 +10262,12 @@ static int do_meta_command(char *zLine, ShellState *p){
       }
     }
     if( bSchema ){
-      zSql = "SELECT lower(name) FROM sqlite_schema"
+      zSql = "SELECT lower(name) as tname FROM sqlite_schema"
              " WHERE type='table' AND coalesce(rootpage,0)>1"
              " UNION ALL SELECT 'sqlite_schema'"
              " ORDER BY 1 collate nocase";
     }else{
-      zSql = "SELECT lower(name) FROM sqlite_schema"
+      zSql = "SELECT lower(name) as tname FROM sqlite_schema"
              " WHERE type='table' AND coalesce(rootpage,0)>1"
              " AND name NOT LIKE 'sqlite_%'"
              " ORDER BY 1 collate nocase";
@@ -10327,6 +10328,58 @@ static int do_meta_command(char *zLine, ShellState *p){
     }else{
       shell_exec(p, zSql, 0);
     }
+#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && !defined(SQLITE_OMIT_VIRTUALTABLE)
+    {
+      int lrc;
+      char *zRevText = /* Query for reversible to-blob-to-text check */
+	"SELECT lower(name) as tname FROM sqlite_schema\n"
+	"WHERE type='table' AND coalesce(rootpage,0)>1\n"
+	"AND name NOT LIKE 'sqlite_%%'%s\n"
+	"ORDER BY 1 collate nocase";
+      zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : "");
+      zRevText = sqlite3_mprintf(
+	  /* lower-case query is first run, producing upper-case query. */
+	  "with tabcols as materialized(\n"
+	  "select tname, cname\n"
+	  "from ("
+	  " select ss.tname as tname, ti.name as cname\n"
+	  " from (%z) ss\n inner join pragma_table_info(tname) ti))\n"
+	  "select 'SELECT total(bad_text_count) AS bad_text_count\n"
+	  "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n"
+	  " from (select 'SELECT COUNT(*) AS bad_text_count\n"
+	  "FROM '||tname||' WHERE '\n"
+	  "||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname\n"
+	  "|| ' AND typeof('||cname||')=''text'' ',\n"
+	  "' OR ') as query, tname from tabcols group by tname)"
+	  , zRevText);
+      shell_check_oom(zRevText);
+      if( bDebug ) utf8_printf(p->out, "%s\n", zRevText);
+      lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0);
+      assert(lrc==SQLITE_OK);
+      if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC);
+      lrc = SQLITE_ROW==sqlite3_step(pStmt);
+      if( lrc ){
+        const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0);
+        sqlite3_stmt *pCheckStmt;
+        lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0);
+        if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery);
+        if( SQLITE_OK==lrc ){
+	  if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){
+	    double countIrreversible = sqlite3_column_double(pCheckStmt, 0);
+	    if( countIrreversible>0 ){
+	      int n = (int)(countIrreversible + 0.5);
+	      utf8_printf(stderr,
+		 "Digest includes %d invalidly encoded text field%s.\n",
+		 n, (n>1)? "s": "");
+	    }
+	  }
+	  sqlite3_finalize(pCheckStmt);
+        }
+        sqlite3_finalize(pStmt);
+      }
+      sqlite3_free(zRevText);
+    }
+#endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */
     sqlite3_free(zSql);
   }else
 
@@ -11118,6 +11171,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss,
           goto PlainScan;
         case '`': case '\'': case '"':
           if(*zLine==cWait){
+	    /* Swallow doubled end-delimiter.*/
             ++zLine;
             continue;
           }
@@ -11288,7 +11342,7 @@ static int process_input(ShellState *p){
   }
   ++p->inputNesting;
   p->lineno = 0;
-  CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE);
+  CONTINUE_PROMPT_RESET;
   while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){
     fflush(p->out);
     zLine = one_input_line(p->in, zLine, nSql>0);
@@ -11307,7 +11361,7 @@ static int process_input(ShellState *p){
         && line_is_complete(zSql, nSql) ){
       memcpy(zLine,";",2);
     }
-    qss = quickscan(zLine, qss, CONTINUE_PROMPT_STATE);
+    qss = quickscan(zLine, qss, CONTINUE_PROMPT_PSTATE);
     if( QSS_PLAINWHITE(qss) && nSql==0 ){
       /* Just swallow single-line whitespace */
       echo_group_input(p, zLine);
@@ -11315,7 +11369,7 @@ static int process_input(ShellState *p){
       continue;
     }
     if( zLine && (zLine[0]=='.' || zLine[0]=='#') && nSql==0 ){
-      CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE);
+      CONTINUE_PROMPT_RESET;
       echo_group_input(p, zLine);
       if( zLine[0]=='.' ){
         rc = do_meta_command(zLine, p);
@@ -11351,7 +11405,7 @@ static int process_input(ShellState *p){
     if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){
       echo_group_input(p, zSql);
       errCnt += runOneSqlLine(p, zSql, p->in, startline);
-      CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE);
+      CONTINUE_PROMPT_RESET;
       nSql = 0;
       if( p->outCount ){
         output_reset(p);
@@ -11371,7 +11425,7 @@ static int process_input(ShellState *p){
     /* This may be incomplete. Let the SQL parser deal with that. */
     echo_group_input(p, zSql);
     errCnt += runOneSqlLine(p, zSql, p->in, startline);
-    CONTINUE_PROMPT_RESET(CONTINUE_PROMPT_STATE);
+    CONTINUE_PROMPT_RESET;
   }
   free(zSql);
   free(zLine);
diff --git a/src/test_demovfs.c b/src/test_demovfs.c
index 29307675b..e990e98f2 100644
--- a/src/test_demovfs.c
+++ b/src/test_demovfs.c
@@ -461,7 +461,6 @@ static int demoDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
 
   if( rc==0 && dirSync ){
     int dfd;                      /* File descriptor open on directory */
-    int i;                        /* Iterator variable */
     char *zSlash;
     char zDir[MAXPATHNAME+1];     /* Name of directory containing file zPath */
 
diff --git a/src/vdbe.c b/src/vdbe.c
index 1bf0ceb29..12b22992b 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -790,15 +790,12 @@ int sqlite3VdbeExec(
     assert( pOp>=aOp && pOp<&aOp[p->nOp]);
     nVmStep++;
 #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
-    if( p->anExec ){
-      assert( p->anExec && p->anCycle );
-      p->anExec[(int)(pOp-aOp)]++;
-      pnCycle = &p->anCycle[pOp-aOp];
+    pOp->nExec++;
+    pnCycle = &pOp->nCycle;
 # ifdef VDBE_PROFILE
-      if( sqlite3NProfileCnt==0 )
+    if( sqlite3NProfileCnt==0 )
 # endif
-        *pnCycle -= sqlite3Hwtime();
-    }
+      *pnCycle -= sqlite3Hwtime();
 #endif
 
     /* Only allow tracing if SQLITE_DEBUG is defined.
@@ -2780,7 +2777,7 @@ case OP_Offset: {          /* out3 */
 ** typeof() function or the IS NULL or IS NOT NULL operators or the
 ** equivalent.  In this case, all content loading can be omitted.
 */
-case OP_Column: {
+case OP_Column: {            /* ncycle */
   u32 p2;            /* column number to retrieve */
   VdbeCursor *pC;    /* The VDBE cursor */
   BtCursor *pCrsr;   /* The B-Tree cursor corresponding to pC */
@@ -4132,7 +4129,7 @@ case OP_SetCookie: {
 **
 ** See also: OP_OpenRead, OP_ReopenIdx
 */
-case OP_ReopenIdx: {
+case OP_ReopenIdx: {         /* ncycle */
   int nField;
   KeyInfo *pKeyInfo;
   u32 p2;
@@ -4153,7 +4150,7 @@ case OP_ReopenIdx: {
   }
   /* If the cursor is not currently open or is open on a different
   ** index, then fall through into OP_OpenRead to force a reopen */
-case OP_OpenRead:
+case OP_OpenRead:            /* ncycle */
 case OP_OpenWrite:
 
   assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ );
@@ -4247,7 +4244,7 @@ open_cursor_set_hints:
 **
 ** Duplicate ephemeral cursors are used for self-joins of materialized views.
 */
-case OP_OpenDup: {
+case OP_OpenDup: {           /* ncycle */
   VdbeCursor *pOrig;    /* The original cursor to be duplicated */
   VdbeCursor *pCx;      /* The new cursor */
 
@@ -4309,8 +4306,8 @@ case OP_OpenDup: {
 ** by this opcode will be used for automatically created transient
 ** indices in joins.
 */
-case OP_OpenAutoindex: 
-case OP_OpenEphemeral: {
+case OP_OpenAutoindex:       /* ncycle */
+case OP_OpenEphemeral: {     /* ncycle */
   VdbeCursor *pCx;
   KeyInfo *pKeyInfo;
 
@@ -4468,7 +4465,7 @@ case OP_OpenPseudo: {
 ** Close a cursor previously opened as P1.  If P1 is not
 ** currently open, this instruction is a no-op.
 */
-case OP_Close: {
+case OP_Close: {             /* ncycle */
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]);
   p->apCsr[pOp->p1] = 0;
@@ -4585,10 +4582,10 @@ case OP_ColumnsUsed: {
 **
 ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt
 */
-case OP_SeekLT:         /* jump, in3, group */
-case OP_SeekLE:         /* jump, in3, group */
-case OP_SeekGE:         /* jump, in3, group */
-case OP_SeekGT: {       /* jump, in3, group */
+case OP_SeekLT:         /* jump, in3, group, ncycle */
+case OP_SeekLE:         /* jump, in3, group, ncycle */
+case OP_SeekGE:         /* jump, in3, group, ncycle */
+case OP_SeekGT: {       /* jump, in3, group, ncycle */
   int res;           /* Comparison result */
   int oc;            /* Opcode */
   VdbeCursor *pC;    /* The cursor to seek */
@@ -4854,7 +4851,7 @@ seek_not_found:
 **      jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0.
 ** </ol>
 */
-case OP_SeekScan: {
+case OP_SeekScan: {          /* ncycle */
   VdbeCursor *pC;
   int res;
   int nStep;
@@ -4976,7 +4973,7 @@ case OP_SeekScan: {
 **
 ** P1 must be a valid b-tree cursor.
 */
-case OP_SeekHit: {
+case OP_SeekHit: {           /* ncycle */
   VdbeCursor *pC;
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
@@ -5108,7 +5105,7 @@ case OP_IfNotOpen: {        /* jump */
 **
 ** See also: NotFound, Found, NotExists
 */
-case OP_IfNoHope: {     /* jump, in3 */
+case OP_IfNoHope: {     /* jump, in3, ncycle */
   VdbeCursor *pC;
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
@@ -5122,9 +5119,9 @@ case OP_IfNoHope: {     /* jump, in3 */
   /* Fall through into OP_NotFound */
   /* no break */ deliberate_fall_through
 }
-case OP_NoConflict:     /* jump, in3 */
-case OP_NotFound:       /* jump, in3 */
-case OP_Found: {        /* jump, in3 */
+case OP_NoConflict:     /* jump, in3, ncycle */
+case OP_NotFound:       /* jump, in3, ncycle */
+case OP_Found: {        /* jump, in3, ncycle */
   int alreadyExists;
   int ii;
   VdbeCursor *pC;
@@ -5254,7 +5251,7 @@ case OP_Found: {        /* jump, in3 */
 **
 ** See also: Found, NotFound, NoConflict, SeekRowid
 */
-case OP_SeekRowid: {        /* jump, in3 */
+case OP_SeekRowid: {        /* jump, in3, ncycle */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
@@ -5279,7 +5276,7 @@ case OP_SeekRowid: {        /* jump, in3 */
   }
   /* Fall through into OP_NotExists */
   /* no break */ deliberate_fall_through
-case OP_NotExists:          /* jump, in3 */
+case OP_NotExists:          /* jump, in3, ncycle */
   pIn3 = &aMem[pOp->p3];
   assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid );
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
@@ -5903,7 +5900,7 @@ case OP_RowData: {
 ** be a separate OP_VRowid opcode for use with virtual tables, but this
 ** one opcode now works for both table types.
 */
-case OP_Rowid: {                 /* out2 */
+case OP_Rowid: {                 /* out2, ncycle */
   VdbeCursor *pC;
   i64 v;
   sqlite3_vtab *pVtab;
@@ -6002,8 +5999,8 @@ case OP_NullRow: {
 ** from the end toward the beginning.  In other words, the cursor is
 ** configured to use Prev, not Next.
 */
-case OP_SeekEnd:
-case OP_Last: {        /* jump */
+case OP_SeekEnd:             /* ncycle */
+case OP_Last: {              /* jump, ncycle */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
@@ -6108,7 +6105,7 @@ case OP_Sort: {        /* jump */
 ** from the beginning toward the end.  In other words, the cursor is
 ** configured to use Next, not Prev.
 */
-case OP_Rewind: {        /* jump */
+case OP_Rewind: {        /* jump, ncycle */
   VdbeCursor *pC;
   BtCursor *pCrsr;
   int res;
@@ -6202,7 +6199,7 @@ case OP_SorterNext: {  /* jump */
   rc = sqlite3VdbeSorterNext(db, pC);
   goto next_tail;
 
-case OP_Prev:          /* jump */
+case OP_Prev:          /* jump, ncycle */
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   assert( pOp->p5==0
        || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP
@@ -6217,7 +6214,7 @@ case OP_Prev:          /* jump */
   rc = sqlite3BtreePrevious(pC->uc.pCursor, pOp->p3);
   goto next_tail;
 
-case OP_Next:          /* jump */
+case OP_Next:          /* jump, ncycle */
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   assert( pOp->p5==0
        || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP
@@ -6409,8 +6406,8 @@ case OP_IdxDelete: {
 **
 ** See also: Rowid, MakeRecord.
 */
-case OP_DeferredSeek:
-case OP_IdxRowid: {           /* out2 */
+case OP_DeferredSeek:         /* ncycle */
+case OP_IdxRowid: {           /* out2, ncycle */
   VdbeCursor *pC;             /* The P1 index cursor */
   VdbeCursor *pTabCur;        /* The P2 table cursor (OP_DeferredSeek only) */
   i64 rowid;                  /* Rowid that P1 current points to */
@@ -6472,8 +6469,8 @@ case OP_IdxRowid: {           /* out2 */
 ** seek operation now, without further delay.  If the cursor seek has
 ** already occurred, this instruction is a no-op.
 */
-case OP_FinishSeek: {
-  VdbeCursor *pC;             /* The P1 index cursor */
+case OP_FinishSeek: {        /* ncycle */
+  VdbeCursor *pC;            /* The P1 index cursor */
 
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
@@ -6528,10 +6525,10 @@ case OP_FinishSeek: {
 ** If the P1 index entry is less than or equal to the key value then jump
 ** to P2. Otherwise fall through to the next instruction.
 */
-case OP_IdxLE:          /* jump */
-case OP_IdxGT:          /* jump */
-case OP_IdxLT:          /* jump */
-case OP_IdxGE:  {       /* jump */
+case OP_IdxLE:          /* jump, ncycle */
+case OP_IdxGT:          /* jump, ncycle */
+case OP_IdxLT:          /* jump, ncycle */
+case OP_IdxGE:  {       /* jump, ncycle */
   VdbeCursor *pC;
   int res;
   UnpackedRecord r;
@@ -7152,10 +7149,6 @@ case OP_Program: {        /* jump */
     pFrame->aOp = p->aOp;
     pFrame->nOp = p->nOp;
     pFrame->token = pProgram->token;
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
-    pFrame->anExec = p->anExec;
-    pFrame->anCycle = p->anCycle;
-#endif
 #ifdef SQLITE_DEBUG
     pFrame->iFrameMagic = SQLITE_FRAME_MAGIC;
 #endif
@@ -7192,10 +7185,6 @@ case OP_Program: {        /* jump */
   memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8);
   p->aOp = aOp = pProgram->aOp;
   p->nOp = pProgram->nOp;
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
-  p->anExec = 0;
-  p->anCycle = 0;
-#endif
 #ifdef SQLITE_DEBUG
   /* Verify that second and subsequent executions of the same trigger do not
   ** try to reuse register values from the first use. */
@@ -7952,7 +7941,7 @@ case OP_VDestroy: {
 ** P1 is a cursor number.  This opcode opens a cursor to the virtual
 ** table and stores that cursor in P1.
 */
-case OP_VOpen: {
+case OP_VOpen: {             /* ncycle */
   VdbeCursor *pCur;
   sqlite3_vtab_cursor *pVCur;
   sqlite3_vtab *pVtab;
@@ -7999,7 +7988,7 @@ case OP_VOpen: {
 ** cursor.  Register P3 is used to hold the values returned by
 ** sqlite3_vtab_in_first() and sqlite3_vtab_in_next().
 */
-case OP_VInitIn: {        /* out2 */
+case OP_VInitIn: {        /* out2, ncycle */
   VdbeCursor *pC;         /* The cursor containing the RHS values */
   ValueList *pRhs;        /* New ValueList object to put in reg[P2] */
 
@@ -8036,7 +8025,7 @@ case OP_VInitIn: {        /* out2 */
 **
 ** A jump is made to P2 if the result set after filtering would be empty.
 */
-case OP_VFilter: {   /* jump */
+case OP_VFilter: {   /* jump, ncycle */
   int nArg;
   int iQuery;
   const sqlite3_module *pModule;
@@ -8096,7 +8085,7 @@ case OP_VFilter: {   /* jump */
 ** bits (OPFLAG_LENGTHARG or OPFLAG_TYPEOFARG) but those bits are
 ** unused by OP_VColumn.
 */
-case OP_VColumn: {
+case OP_VColumn: {           /* ncycle */
   sqlite3_vtab *pVtab;
   const sqlite3_module *pModule;
   Mem *pDest;
@@ -8148,7 +8137,7 @@ case OP_VColumn: {
 ** jump to instruction P2.  Or, if the virtual table has reached
 ** the end of its result set, then fall through to the next instruction.
 */
-case OP_VNext: {   /* jump */
+case OP_VNext: {   /* jump, ncycle */
   sqlite3_vtab *pVtab;
   const sqlite3_module *pModule;
   int res;
@@ -8732,17 +8721,11 @@ default: {          /* This is really OP_Noop, OP_Explain */
     }
 
 #if defined(VDBE_PROFILE)
-    assert( pnCycle );
-    if( pnCycle ){
-      *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
-      assert( *pnCycle < (((u64)1) << 48) );
-      pnCycle = 0;
-    }
+    *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
+    pnCycle = 0;
 #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS)
-    if( pnCycle ){
-      *pnCycle += sqlite3Hwtime();
-      pnCycle = 0;
-    }
+    *pnCycle += sqlite3Hwtime();
+    pnCycle = 0;
 #endif
 
     /* The following code adds nothing to the actual functionality
@@ -8819,11 +8802,10 @@ abort_due_to_error:
   ** top. */
 vdbe_return:
 #if defined(VDBE_PROFILE)
-    if( pnCycle ){
-      *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
-      assert( *pnCycle < (((u64)1) << 48) );
-      pnCycle = 0;
-    }
+  if( pnCycle ){
+    *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
+    pnCycle = 0;
+  }
 #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS)
   if( pnCycle ){
     *pnCycle += sqlite3Hwtime();
diff --git a/src/vdbe.h b/src/vdbe.h
index 467c99934..17a301bd8 100644
--- a/src/vdbe.h
+++ b/src/vdbe.h
@@ -71,6 +71,10 @@ struct VdbeOp {
   u32 iSrcLine;            /* Source-code line that generated this opcode
                            ** with flags in the upper 8 bits */
 #endif
+#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
+  u64 nExec;
+  u64 nCycle;
+#endif
 };
 typedef struct VdbeOp VdbeOp;
 
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index df47e0544..cd112e9c4 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -171,8 +171,6 @@ struct VdbeFrame {
   Vdbe *v;                /* VM this frame belongs to */
   VdbeFrame *pParent;     /* Parent of this frame, or NULL if parent is main */
   Op *aOp;                /* Program instructions for parent frame */
-  i64 *anExec;            /* Event counters from parent frame */
-  u64 *anCycle;           /* Cycle counters from parent frame */
   Mem *aMem;              /* Array of memory cells for parent frame */
   VdbeCursor **apCsr;     /* Array of Vdbe cursors for parent frame */
   u8 *aOnce;              /* Bitmask used by OP_Once */
@@ -492,10 +490,6 @@ struct Vdbe {
   u32 expmask;            /* Binding to these vars invalidates VM */
   SubProgram *pProgram;   /* Linked list of all sub-programs used by VM */
   AuxData *pAuxData;      /* Linked list of auxdata allocations */
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
-  i64 *anExec;            /* Number of times each op has been executed */
-  u64 *anCycle;           /* Sum of sqlite3HwTime() spent in each opcode */
-#endif
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   int nScan;              /* Entries in aScan[] */
   ScanStatus *aScan;      /* Scan definitions for sqlite3_stmt_scanstatus() */
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index 0644cffa5..414745e8b 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -15,6 +15,7 @@
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
+#include "opcodes.h"
 
 #ifndef SQLITE_OMIT_DEPRECATED
 /*
@@ -2122,14 +2123,12 @@ int sqlite3_stmt_scanstatus_v2(
   ScanStatus *pScan;
   int idx;
 
-  /* If the v2 flag is clear, then this function must ignore any ScanStatus
-  ** structures with ScanStatus.addrLoop set to 0. */
   if( iScan<0 ){
     int ii;
     if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){
       i64 res = 0;
       for(ii=0; ii<p->nOp; ii++){
-        res += p->anCycle[ii];
+        res += p->aOp[ii].nCycle;
       }
       *(i64*)pOut = res;
       return 0;
@@ -2140,6 +2139,8 @@ int sqlite3_stmt_scanstatus_v2(
     idx = iScan;
     pScan = &p->aScan[idx];
   }else{
+    /* If the COMPLEX flag is clear, then this function must ignore any 
+    ** ScanStatus structures with ScanStatus.addrLoop set to 0. */
     for(idx=0; idx<p->nScan; idx++){
       pScan = &p->aScan[idx];
       if( pScan->zName ){
@@ -2153,7 +2154,7 @@ int sqlite3_stmt_scanstatus_v2(
   switch( iScanStatusOp ){
     case SQLITE_SCANSTAT_NLOOP: {
       if( pScan->addrLoop>0 ){
-        *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop];
+        *(sqlite3_int64*)pOut = p->aOp[pScan->addrLoop].nExec;
       }else{
         *(sqlite3_int64*)pOut = -1;
       }
@@ -2161,7 +2162,7 @@ int sqlite3_stmt_scanstatus_v2(
     }
     case SQLITE_SCANSTAT_NVISIT: {
       if( pScan->addrVisit>0 ){
-        *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit];
+        *(sqlite3_int64*)pOut = p->aOp[pScan->addrVisit].nExec;
       }else{
         *(sqlite3_int64*)pOut = -1;
       }
@@ -2217,7 +2218,7 @@ int sqlite3_stmt_scanstatus_v2(
           if( iIns==0 ) break;
           if( iIns>0 ){
             while( iIns<=iEnd ){
-              res += p->anCycle[iIns];
+              res += p->aOp[iIns].nCycle;
               iIns++;
             }
           }else{
@@ -2225,13 +2226,10 @@ int sqlite3_stmt_scanstatus_v2(
             for(iOp=0; iOp<p->nOp; iOp++){
               Op *pOp = &p->aOp[iOp];
               if( pOp->p1!=iEnd ) continue;
-              if( pOp->opcode!=OP_VFilter && pOp->opcode!=OP_VColumn
-               && pOp->opcode!=OP_Rowid   && pOp->opcode!=OP_VOpen
-               && pOp->opcode!=OP_VNext
-              ){
+              if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){
                 continue;
               }
-              res += p->anCycle[iOp];
+              res += p->aOp[iOp].nCycle;
             }
           }
         }
@@ -2263,7 +2261,11 @@ int sqlite3_stmt_scanstatus(
 */
 void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){
   Vdbe *p = (Vdbe*)pStmt;
-  memset(p->anExec, 0, p->nOp * sizeof(i64));
-  memset(p->anCycle, 0, p->nOp * sizeof(u64));
+  int ii;
+  for(ii=0; ii<p->nOp; ii++){
+    Op *pOp = &p->aOp[ii];
+    pOp->nExec = 0;
+    pOp->nCycle = 0;
+  }
 }
 #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index e22b74119..8d284746f 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -262,6 +262,10 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){
 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
   pOp->zComment = 0;
 #endif
+#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
+  pOp->nExec = 0;
+  pOp->nCycle = 0;
+#endif
 #ifdef SQLITE_DEBUG
   if( p->db->flags & SQLITE_VdbeAddopTrace ){
     sqlite3VdbePrintOp(0, i, &p->aOp[i]);
@@ -2508,8 +2512,10 @@ void sqlite3VdbeRewind(Vdbe *p){
   p->iStatement = 0;
   p->nFkConstraint = 0;
 #ifdef VDBE_PROFILE
-  memset(p->anExec, 0, sizeof(i64)*p->nOp);
-  memset(p->anCycle, 0, sizeof(u64)*p->nOp);
+  for(i=0; i<p->nOp; i++){
+    p->aOp[i].nExec = 0;
+    p->aOp[i].nCycle = 0;
+  }
 #endif
 }
 
@@ -2617,10 +2623,6 @@ void sqlite3VdbeMakeReady(
   p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem));
   p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*));
   p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*));
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
-  p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64));
-  p->anCycle = allocSpace(&x, 0, p->nOp*sizeof(u64));
-#endif
   if( x.nNeeded ){
     x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded);
     x.nFree = x.nNeeded;
@@ -2629,10 +2631,6 @@ void sqlite3VdbeMakeReady(
       p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem));
       p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*));
       p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*));
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
-      p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64));
-      p->anCycle = allocSpace(&x, p->anCycle, p->nOp*sizeof(u64));
-#endif
     }
   }
 
@@ -2647,10 +2645,6 @@ void sqlite3VdbeMakeReady(
     p->nMem = nMem;
     initMemArray(p->aMem, nMem, db, MEM_Undefined);
     memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*));
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
-    memset(p->anExec, 0, p->nOp*sizeof(i64));
-    memset(p->anCycle, 0, p->nOp*sizeof(u64));
-#endif
   }
   sqlite3VdbeRewind(p);
 }
@@ -2708,10 +2702,6 @@ static void closeCursorsInFrame(Vdbe *p){
 int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){
   Vdbe *v = pFrame->v;
   closeCursorsInFrame(v);
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
-  v->anExec = pFrame->anExec;
-  v->anCycle = pFrame->anCycle;
-#endif
   v->aOp = pFrame->aOp;
   v->nOp = pFrame->nOp;
   v->aMem = pFrame->aMem;
@@ -3543,8 +3533,8 @@ int sqlite3VdbeReset(Vdbe *p){
       }
       for(i=0; i<p->nOp; i++){
         char zHdr[100];
-        i64 cnt = p->anExec[i];
-        i64 cycles = p->anCycle[i];
+        i64 cnt = p->aOp[i].nExec;
+        i64 cycles = p->aOp[i].nCycle;
         sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ",
            cnt,
            cycles,
diff --git a/src/where.c b/src/where.c
index a13379947..25ad18b28 100644
--- a/src/where.c
+++ b/src/where.c
@@ -67,7 +67,7 @@ int sqlite3WhereIsDistinct(WhereInfo *pWInfo){
 ** block sorting is required.
 */
 int sqlite3WhereIsOrdered(WhereInfo *pWInfo){
-  return pWInfo->nOBSat;
+  return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat;
 }
 
 /*
@@ -812,6 +812,51 @@ static int termCanDriveIndex(
 
 
 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
+
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+/*
+** Argument pIdx represents an automatic index that the current statement
+** will create and populate. Add an OP_Explain with text of the form:
+**
+**     CREATE AUTOMATIC INDEX ON <table>(<cols>) [WHERE <expr>]
+**
+** This is only required if sqlite3_stmt_scanstatus() is enabled, to
+** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP
+** values with. In order to avoid breaking legacy code and test cases, 
+** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command.
+*/
+static void explainAutomaticIndex(
+  Parse *pParse,
+  Index *pIdx,                    /* Automatic index to explain */
+  int bPartial,                   /* True if pIdx is a partial index */
+  int *pAddrExplain               /* OUT: Address of OP_Explain */
+){
+  if( pParse->explain!=2 ){
+    Table *pTab = pIdx->pTable;
+    const char *zSep = "";
+    char *zText = 0;
+    int ii = 0;
+    zText = sqlite3_mprintf("CREATE AUTOMATIC INDEX ON %s(", pTab->zName);
+    assert( pIdx->nColumn>1 );
+    assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID );
+    for(ii=0; ii<(pIdx->nColumn-1); ii++){
+      const char *zName = 0;
+      int iCol = pIdx->aiColumn[ii];
+
+      zName = pTab->aCol[iCol].zCnName;
+      zText = sqlite3_mprintf("%z%s%s", zText, zSep, zName);
+      zSep = ", ";
+    }
+    *pAddrExplain = sqlite3VdbeExplain(
+        pParse, 0, "%s)%s", zText, (bPartial ? " WHERE <expr>" : "")
+    );
+    sqlite3_free(zText);
+  }
+}
+#else
+# define explainAutomaticIndex(a,b,c,d)
+#endif
+
 /*
 ** Generate code to construct the Index object for an automatic index
 ** and to set up the WhereLevel object pLevel so that the code generator
@@ -847,6 +892,9 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   SrcItem *pTabItem;          /* FROM clause term being indexed */
   int addrCounter = 0;        /* Address where integer counter is initialized */
   int regBase;                /* Array of registers where record is assembled */
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+  int addrExp = 0;            /* Address of OP_Explain */
+#endif
 
   /* Generate code to skip over the creation and initialization of the
   ** transient index on 2nd and subsequent iterations of the loop. */
@@ -970,6 +1018,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   pIdx->azColl[n] = sqlite3StrBINARY;
 
   /* Create the automatic index */
+  explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp);
   assert( pLevel->iIdxCur>=0 );
   pLevel->iIdxCur = pParse->nTab++;
   sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
@@ -1005,6 +1054,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
     sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0,
                          regBase, pLoop->u.btree.nEq);
   }
+  sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v));
   sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
   sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
   if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
@@ -1025,6 +1075,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   
   /* Jump here when skipping the initialization */
   sqlite3VdbeJumpHere(v, addrInit);
+  sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1);
 
 end_auto_index_create:
   sqlite3ExprDelete(pParse->db, pPartial);
diff --git a/src/wherecode.c b/src/wherecode.c
index 54c79baf9..4c22e5dd6 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -294,16 +294,26 @@ void sqlite3WhereAddScanStatus(
 ){
   const char *zObj = 0;
   WhereLoop *pLoop = pLvl->pWLoop;
-  if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0  &&  pLoop->u.btree.pIndex!=0 ){
+  int wsFlags = pLoop->wsFlags;
+  int viaCoroutine = 0;
+
+  if( (wsFlags & WHERE_VIRTUALTABLE)==0  &&  pLoop->u.btree.pIndex!=0 ){
     zObj = pLoop->u.btree.pIndex->zName;
   }else{
     zObj = pSrclist->a[pLvl->iFrom].zName;
+    viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine;
   }
   sqlite3VdbeScanStatus(
       v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj
   );
-  if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){
-    sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur);
+
+  if( viaCoroutine==0 ){
+    if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){
+      sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur);
+    }
+    if( wsFlags & WHERE_INDEXED ){
+      sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur);
+    }
   }
 }
 #endif