Do not activate the truthProb adjustment mechanism if the truth probability

is less than the heuristic value, as there could be correlations unknown to
stat4.  Also add additional tracing output to make truthProb adjustments more
visible.

FossilOrigin-Name: c535fea147ce5c6e4aab25d3c85a3f53a7364c5b5ee10fb6d393c5911a02be7e

3 files changed, 40 insertions, 23 deletions

diff --git a/manifest b/manifest
index e68e44817..9f902b184 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\snew\stest\sfile\sanalyzeG.test,\scontaining\sa\stest\sfor\sthe\schange\son\sthis\sbranch.
-D 2020-02-22T17:32:00.691
+C Do\snot\sactivate\sthe\struthProb\sadjustment\smechanism\sif\sthe\struth\sprobability\nis\sless\sthan\sthe\sheuristic\svalue,\sas\sthere\scould\sbe\scorrelations\sunknown\sto\nstat4.\s\sAlso\sadd\sadditional\stracing\soutput\sto\smake\struthProb\sadjustments\smore\nvisible.
+D 2020-02-22T18:27:48.175
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -617,7 +617,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 697424314e40d99f93f548c7bfa526c10e87f4bdf64d5a76a96b999dd7133ebc
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c a137468bf36c92e64d2275caa80c83902e3a0fc59273591b96c6416d3253d05d
-F src/where.c 74a2fc5a900eab9a2fdda2017a290f0eeaa9c5597fdb86322ea2ccbc3758c71d
+F src/where.c 44695e878a287d8c1d4976e2e85bea29994facec3972beb7ca22437d62cda6a5
 F src/whereInt.h 94e3aadcf43b4d16279182d147c9e4f8ef6ed5a5bd1ecc021639c29336b0a3eb
 F src/wherecode.c f5df56e395ade2240cabb2d39500c681bd29f8cc0636c3301c4996ad160df94d
 F src/whereexpr.c 264d58971eaf8256eb5b0917bcd7fc7a1f1109fdda183a8382308a1b18a2dce7
@@ -1859,7 +1859,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1babd6ec5d60e2c34aa1c0285ead768a88004218468e97262411973fe3487022
-R 2b88c80f8c57b2c7447cf9a6931d1437
-U dan
-Z e854d74d6063f88b5d388938dfbba4ed
+P 243ab1852a2291595527ea1f26e78ad83eda285ae28f876bc1c703677f495cfa
+R 2719b1105cf4d47b954b372c20702e91
+U drh
+Z e3872ca3639b1687907dbcae881883fa
diff --git a/manifest.uuid b/manifest.uuid
index 2b78df281..e6f868ae7 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-243ab1852a2291595527ea1f26e78ad83eda285ae28f876bc1c703677f495cfa
\ No newline at end of file
+c535fea147ce5c6e4aab25d3c85a3f53a7364c5b5ee10fb6d393c5911a02be7e
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 58cf11ab1..03bfad3b7 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2305,7 +2305,7 @@ static void whereLoopOutputAdjust(
           if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
             k = 10;
           }else{
-            k = 20;
+            k = 20;  /* Keep the "20" value in sync. See tag-20200222-1 */
           }
           if( iReduce<k ){
             pTerm->wtFlags |= TERM_HEURTRUTH;
@@ -2659,7 +2659,13 @@ static int whereLoopAddBtreeIndex(
           if( rc!=SQLITE_OK ) break;          /* Jump out of the pTerm loop */
           if( nOut ){
             pNew->nOut = sqlite3LogEst(nOut);
-            if( nEq==1 && pTerm->truthProb>0 ){
+            if( nEq==1
+             && pTerm->truthProb>0
+             /* TUNING: Adjust truthProb from the default heuristic only if the
+             ** probability is close to 1.0. The "20" constant is copied from
+             ** the heuristic at tag-20200222-1. Keep values in sync */
+             && pNew->nOut+20 > pProbe->aiRowLogEst[0]
+            ){
 #if WHERETRACE_ENABLED /* 0x01 */
               if( sqlite3WhereTrace & 0x01 ){
                 sqlite3DebugPrintf("Update truthProb from %d to %d:\n",
@@ -4555,6 +4561,28 @@ static int exprIsDeterministic(Expr *p){
   return w.eCode;
 }
 
+  
+#ifdef WHERETRACE_ENABLED
+/*
+** Display all WhereLoops in pWInfo
+*/
+static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
+  if( sqlite3WhereTrace ){    /* Display all of the WhereLoop objects */
+    WhereLoop *p;
+    int i;
+    static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
+                                           "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
+    for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
+      p->cId = zLabel[i%(sizeof(zLabel)-1)];
+      sqlite3WhereLoopPrint(p, pWC);
+    }
+  }
+}
+# define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C)
+#else
+# define WHERETRACE_ALL_LOOPS(W,C)
+#endif
+
 /*
 ** Generate the beginning of the loop used for WHERE clause processing.
 ** The return value is a pointer to an opaque structure that contains
@@ -4864,6 +4892,7 @@ WhereInfo *sqlite3WhereBegin(
     ** then we need to rerun the whole loop building process so that all
     ** loops will be built using the revised truthProb values. */
     if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
+      WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
       WHERETRACE(0xffff, 
          ("**** Redo all loop computations due to truthProb changes ****\n"));
       while( pWInfo->pLoops ){
@@ -4875,19 +4904,7 @@ WhereInfo *sqlite3WhereBegin(
       if( rc ) goto whereBeginError;
     }
 #endif
-  
-#ifdef WHERETRACE_ENABLED
-    if( sqlite3WhereTrace ){    /* Display all of the WhereLoop objects */
-      WhereLoop *p;
-      int i;
-      static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz"
-                                             "ABCDEFGHIJKLMNOPQRSTUVWYXZ";
-      for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){
-        p->cId = zLabel[i%(sizeof(zLabel)-1)];
-        sqlite3WhereLoopPrint(p, sWLB.pWC);
-      }
-    }
-#endif
+    WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC);
   
     wherePathSolver(pWInfo, 0);
     if( db->mallocFailed ) goto whereBeginError;