More precision in comparing integers and floating point values while

processing the integer primary key for OP_SeekGE and similar.
[forum:/forumpost/2bdb86a068|Forum post 2bdb86a068].

FossilOrigin-Name: f9c6426de3b413ff8fcf04a00931ca5f123f996c572b35181af114afa8d811d7

3 files changed, 7 insertions, 4 deletions

diff --git a/src/vdbe.c b/src/vdbe.c
index 5b42976a0..cfcd62266 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -4275,6 +4275,7 @@ case OP_SeekGT: {       /* jump, in3, group */
     /* If the P3 value could not be converted into an integer without
     ** loss of information, then special processing is required... */
     if( (newType & (MEM_Int|MEM_IntReal))==0 ){
+      int c;
       if( (newType & MEM_Real)==0 ){
         if( (newType & MEM_Null) || oc>=OP_SeekGE ){
           VdbeBranchTaken(1,2);
@@ -4284,7 +4285,8 @@ case OP_SeekGT: {       /* jump, in3, group */
           if( rc!=SQLITE_OK ) goto abort_due_to_error;
           goto seek_not_found;
         }
-      }else
+      }
+      c = sqlite3IntFloatCompare(iKey, pIn3->u.r);
 
       /* If the approximation iKey is larger than the actual real search
       ** term, substitute >= for > and < for <=. e.g. if the search term
@@ -4293,7 +4295,7 @@ case OP_SeekGT: {       /* jump, in3, group */
       **        (x >  4.9)    ->     (x >= 5)
       **        (x <= 4.9)    ->     (x <  5)
       */
-      if( pIn3->u.r<(double)iKey ){
+      if( c>0 ){
         assert( OP_SeekGE==(OP_SeekGT-1) );
         assert( OP_SeekLT==(OP_SeekLE-1) );
         assert( (OP_SeekLE & 0x0001)==(OP_SeekGT & 0x0001) );
@@ -4302,7 +4304,7 @@ case OP_SeekGT: {       /* jump, in3, group */
 
       /* If the approximation iKey is smaller than the actual real search
       ** term, substitute <= for < and > for >=.  */
-      else if( pIn3->u.r>(double)iKey ){
+      else if( c<0 ){
         assert( OP_SeekLE==(OP_SeekLT+1) );
         assert( OP_SeekGT==(OP_SeekGE+1) );
         assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) );
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 4a41c43a5..763c4ed09 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -533,6 +533,7 @@ int sqlite3VdbeMemIsRowSet(const Mem*);
 int sqlite3VdbeMemSetRowSet(Mem*);
 int sqlite3VdbeMemMakeWriteable(Mem*);
 int sqlite3VdbeMemStringify(Mem*, u8, u8);
+int sqlite3IntFloatCompare(i64,double);
 i64 sqlite3VdbeIntValue(Mem*);
 int sqlite3VdbeMemIntegerify(Mem*);
 double sqlite3VdbeRealValue(Mem*);
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 7f08b5d25..5c9ab11c7 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -4285,7 +4285,7 @@ SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){
 ** number.  Return negative, zero, or positive if the first (i64) is less than,
 ** equal to, or greater than the second (double).
 */
-static int sqlite3IntFloatCompare(i64 i, double r){
+int sqlite3IntFloatCompare(i64 i, double r){
   if( sizeof(LONGDOUBLE_TYPE)>8 ){
     LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i;
     testcase( x<r );