Enforce consistent comparison results between integer and floating point

values.  Fix for ticket [38a97a87a6e4e83].

FossilOrigin-Name: 849e686da2d20fd7f938427475fc792384d2aa7c

6 files changed, 150 insertions, 45 deletions

diff --git a/manifest b/manifest
index 075ff7fbd..96707680e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Test\scases\sfor\sthe\sieee754\sextension.
-D 2015-11-06T17:01:20.579
+C Enforce\sconsistent\scomparison\sresults\sbetween\sinteger\sand\sfloating\spoint\nvalues.\s\sFix\sfor\sticket\s[38a97a87a6e4e83].
+D 2015-11-06T19:53:09.863
 F Makefile.in 3a705bb4bd12e194212ddbdbf068310d17153cdb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 702d3e98f3afc6587a78481257f3c4c900efc3a4
@@ -350,7 +350,7 @@ F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46
 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba
 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e
 F src/tclsqlite.c d9439b6a910985b7fff43ba6756bcef00de22649
-F src/test1.c 9ac5cbfe3c859ab7518edc5109a2959d6bf7b059
+F src/test1.c a719afff3144f7f01c6dc3f7d118ac31d15e7527
 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b
 F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f
 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e
@@ -407,7 +407,7 @@ F src/vdbe.c dfbaae2570172c523bce14299021d352b8508f7f
 F src/vdbe.h efb7a8c1459e31f3ea4377824c6a7e4cb5068637
 F src/vdbeInt.h 33403622c6a8feaaac5f0f3f17f5d1bf6df42286
 F src/vdbeapi.c 020681b943e77766b32ae1cddf86d7831b7374ca
-F src/vdbeaux.c 717bc2ae9cc0b58ef398fff44bd9164bfd96772a
+F src/vdbeaux.c 869c95b0fc73026da1ca179e053807f563793e8f
 F src/vdbeblob.c 565fabd302f5fca3bdf3d56cac330483616a39b6
 F src/vdbemem.c fdd1578e47bea61390d472de53c565781d81e045
 F src/vdbesort.c 8b23930a1289526f6d2a3a9f2e965bcc963e4a68
@@ -453,7 +453,7 @@ F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b
 F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7
 F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a
 F test/async5.test 383ab533fdb9f7ad228cc99ee66e1acb34cc0dc0
-F test/atof1.test 08a61df9365c341f334a65f4348897312d8f3db7
+F test/atof1.test ff0b0156fd705b67c506e1f2bfe9e26102bea9bd
 F test/attach.test 437107943f14d131cf5efc2ae5305a94d7cb1d58
 F test/attach2.test 0ec5defa340363de6cd50fd595046465e9aaba2d
 F test/attach3.test 359eb65d00102cdfcef6fa4e81dc1648f8f80b27
@@ -903,6 +903,7 @@ F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934
 F test/notnull.test f8fcf58669ddba79274daa2770d61dfad8274f62
 F test/null.test 0dcce4f04284ec66108c503327ad6d224c0752b3
 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1
+F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823
 F test/offset1.test f06b83657bcf26f9ce805e67450e189e282143b2
 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394
 F test/orderby1.test 870e150450437d3980badbde3d0166b81d9e33f6
@@ -1399,7 +1400,8 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 33e6606f5e497e81119ec491cf2370f60bddafc0
-R 6a48cd86f6e2c59a9972f4901e737c7d
+P 840cbda88675b6012dae2423252bf42d0d563874 7a9988d38cd7ab3df79eed97631ff23d5928b4ad
+R 281cdf2e28b38f35a9e781fe4ef44557
+T +closed 7a9988d38cd7ab3df79eed97631ff23d5928b4ad
 U drh
-Z 4f550f3fed6ca631b416aee9408c87ae
+Z 043f003e4fd28221190bc705626ad8a1
diff --git a/manifest.uuid b/manifest.uuid
index 5f0e15612..446605d12 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-840cbda88675b6012dae2423252bf42d0d563874
\ No newline at end of file
+849e686da2d20fd7f938427475fc792384d2aa7c
\ No newline at end of file
diff --git a/src/test1.c b/src/test1.c
index dd190feeb..7ce4ed5a5 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -7039,6 +7039,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
 
   };
   static int bitmask_size = sizeof(Bitmask)*8;
+  static int longdouble_size = sizeof(LONGDOUBLE_TYPE);
   int i;
   extern int sqlite3_sync_count, sqlite3_fullsync_count;
   extern int sqlite3_opentemp_count;
@@ -7139,6 +7140,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
       (char*)&sqlite3_data_directory, TCL_LINK_STRING);
   Tcl_LinkVar(interp, "bitmask_size",
       (char*)&bitmask_size, TCL_LINK_INT|TCL_LINK_READ_ONLY);
+  Tcl_LinkVar(interp, "longdouble_size",
+      (char*)&longdouble_size, TCL_LINK_INT|TCL_LINK_READ_ONLY);
   Tcl_LinkVar(interp, "sqlite_sync_count",
       (char*)&sqlite3_sync_count, TCL_LINK_INT);
   Tcl_LinkVar(interp, "sqlite_fullsync_count",
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index f605ea3ec..196a6765d 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -3634,6 +3634,34 @@ static SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){
   return pB1->n - pB2->n;
 }
 
+/*
+** Do a comparison between a 64-bit signed integer and a 64-bit floating-point
+** number.  Return negative, zero, or positive if the first (i64) is less than,
+** equal to, or greater than the second (double).
+*/
+static int sqlite3IntFloatCompare(i64 i, double r){
+  if( sizeof(LONGDOUBLE_TYPE)>8 ){
+    LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i;
+    if( x<r ) return -1;
+    if( x>r ) return +1;
+    return 0;
+  }else{
+    i64 y;
+    double s;
+    if( r<-9223372036854775808.0 ) return +1;
+    if( r>9223372036854775807.0 ) return -1;
+    y = (i64)r;
+    if( i<y ) return -1;
+    if( i>y ){
+      if( y==SMALLEST_INT64 && r>0.0 ) return -1;
+      return +1;
+    }
+    s = (double)i;
+    if( s<r ) return -1;
+    if( s>r ) return +1;
+    return 0;
+  }
+}
 
 /*
 ** Compare the values contained by the two memory cells, returning
@@ -3660,34 +3688,34 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){
     return (f2&MEM_Null) - (f1&MEM_Null);
   }
 
-  /* If one value is a number and the other is not, the number is less.
-  ** If both are numbers, compare as reals if one is a real, or as integers
-  ** if both values are integers.
+  /* At least one of the two values is a number
   */
   if( combined_flags&(MEM_Int|MEM_Real) ){
-    double r1, r2;
     if( (f1 & f2 & MEM_Int)!=0 ){
       if( pMem1->u.i < pMem2->u.i ) return -1;
-      if( pMem1->u.i > pMem2->u.i ) return 1;
+      if( pMem1->u.i > pMem2->u.i ) return +1;
       return 0;
     }
-    if( (f1&MEM_Real)!=0 ){
-      r1 = pMem1->u.r;
-    }else if( (f1&MEM_Int)!=0 ){
-      r1 = (double)pMem1->u.i;
-    }else{
-      return 1;
+    if( (f1 & f2 & MEM_Real)!=0 ){
+      if( pMem1->u.r < pMem2->u.r ) return -1;
+      if( pMem1->u.r > pMem2->u.r ) return +1;
+      return 0;
     }
-    if( (f2&MEM_Real)!=0 ){
-      r2 = pMem2->u.r;
-    }else if( (f2&MEM_Int)!=0 ){
-      r2 = (double)pMem2->u.i;
-    }else{
-      return -1;
+    if( (f1&MEM_Int)!=0 ){
+      if( (f2&MEM_Real)!=0 ){
+        return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r);
+      }else{
+        return -1;
+      }
     }
-    if( r1<r2 ) return -1;
-    if( r1>r2 ) return 1;
-    return 0;
+    if( (f1&MEM_Real)!=0 ){
+      if( (f2&MEM_Int)!=0 ){
+        return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r);
+      }else{
+        return -1;
+      }
+    }
+    return +1;
   }
 
   /* If one value is a string and the other is a blob, the string is less.
@@ -3838,13 +3866,8 @@ int sqlite3VdbeRecordCompareWithSkip(
       }else if( serial_type==0 ){
         rc = -1;
       }else if( serial_type==7 ){
-        double rhs = (double)pRhs->u.i;
         sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
-        if( mem1.u.r<rhs ){
-          rc = -1;
-        }else if( mem1.u.r>rhs ){
-          rc = +1;
-        }
+        rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r);
       }else{
         i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]);
         i64 rhs = pRhs->u.i;
@@ -3868,18 +3891,15 @@ int sqlite3VdbeRecordCompareWithSkip(
       }else if( serial_type==0 ){
         rc = -1;
       }else{
-        double rhs = pRhs->u.r;
-        double lhs;
         sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
         if( serial_type==7 ){
-          lhs = mem1.u.r;
+          if( mem1.u.r<pRhs->u.r ){
+            rc = -1;
+          }else if( mem1.u.r>pRhs->u.r ){
+            rc = +1;
+          }
         }else{
-          lhs = (double)mem1.u.i;
-        }
-        if( lhs<rhs ){
-          rc = -1;
-        }else if( lhs>rhs ){
-          rc = +1;
+          rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r);
         }
       }
     }
@@ -4088,6 +4108,7 @@ static int vdbeRecordCompareString(
   int serial_type;
   int res;
 
+  assert( pPKey2->aMem[0].flags & MEM_Str );
   vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo);
   getVarint32(&aKey1[1], serial_type);
   if( serial_type<12 ){
diff --git a/test/atof1.test b/test/atof1.test
index 5c04d0229..55f5e44d9 100644
--- a/test/atof1.test
+++ b/test/atof1.test
@@ -15,7 +15,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-if {![info exists __GNUC__] || [regexp arm $tcl_platform(machine)]} {
+if {$::longdouble_size<=8} {
   finish_test
   return
 }
diff --git a/test/numindex1.test b/test/numindex1.test
new file mode 100644
index 000000000..c647fc540
--- /dev/null
+++ b/test/numindex1.test
@@ -0,0 +1,79 @@
+# 2015-11-05
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements tests for indexes on large numeric values.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+
+# Test cases from Zsbán Ambrus:
+#
+do_execsql_test numindex1-1.1 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  CREATE INDEX t1b ON t1(b);
+  INSERT INTO t1(a,b) VALUES(100, 356282677878746339);
+  INSERT INTO t1(a,b) VALUES(50, 356282677878746339.0);
+  INSERT INTO t1(a,b) VALUES(0, 356282677878746340);
+  DELETE FROM t1 WHERE a=50;
+  PRAGMA integrity_check;
+} {ok}
+
+do_execsql_test numindex1-1.2 {
+  CREATE TABLE t2(a,b);
+  INSERT INTO t2(a,b) VALUES('b', 1<<58),
+      ('c', (1<<58)+1e-7), ('d', (1<<58)+1);
+  SELECT a, b, typeof(b), '|' FROM t2 ORDER BY +a;
+} {b 288230376151711744 integer | c 2.88230376151712e+17 real | d 288230376151711745 integer |}
+
+do_execsql_test numindex1-1.3 {
+  SELECT x.a || CASE WHEN x.b==y.b THEN '==' ELSE '<>' END || y.a
+    FROM t2 AS x, t2 AS y
+   ORDER BY +x.a, +x.b;
+} {b==b b==c b<>d c==b c==c c<>d d<>b d<>c d==d}
+
+# New test cases
+#
+do_execsql_test numindex1-2.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY,b);
+  CREATE INDEX t1b ON t1(b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100)
+  INSERT INTO t1(a,b) SELECT x, 10000000000000004.0 FROM c
+   WHERE x NOT IN (23,37);
+  INSERT INTO t1(a,b) VALUES(23,10000000000000005);
+  INSERT INTO t1(a,b) VALUES(37,10000000000000003);
+  DELETE FROM t1 WHERE a NOT IN (23,37);
+  PRAGMA integrity_check;
+} {ok}
+
+do_execsql_test numindex1-3.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(a INTEGER PRIMARY KEY,b);
+  CREATE INDEX t1b ON t1(b);
+  WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<20)
+  INSERT INTO t1(a,b) SELECT x, 100000000000000005.0
+    FROM c WHERE x NOT IN (3,5,7,11,13,17,19);
+  INSERT INTO t1(a,b) VALUES(3,100000000000000005);
+  INSERT INTO t1(a,b) VALUES(5,100000000000000000);
+  INSERT INTO t1(a,b) VALUES(7,100000000000000008);
+  INSERT INTO t1(a,b) VALUES(11,100000000000000006);
+  INSERT INTO t1(a,b) VALUES(13,100000000000000001);
+  INSERT INTO t1(a,b) VALUES(17,100000000000000004);
+  INSERT INTO t1(a,b) VALUES(19,100000000000000003);
+  PRAGMA integrity_check;
+} {ok}
+
+do_execsql_test numindex1-3.2 {
+  SELECT a FROM t1 ORDER BY b;
+} {1 2 4 5 6 8 9 10 12 14 15 16 18 20 13 19 17 3 11 7}
+
+finish_test