Repair bug noted by Paul Caskey: neqsel() has been generating a bogus

result, in fact nearly the opposite of what it should, because it
was passing the not-equal operator to eqsel() which would use it to
compare the value against the most common value in the column, and
of course obtain the wrong result therefrom.  Must pass the equality
operator to eqsel() instead.  Fortunately that's easy to get from
the oprnegate link.

1 files changed, 58 insertions, 5 deletions

diff --git a/src/backend/utils/adt/selfuncs.c b/src/backend/utils/adt/selfuncs.c
index 04d9d9f6a9e..16b7964392e 100644
--- a/src/backend/utils/adt/selfuncs.c
+++ b/src/backend/utils/adt/selfuncs.c
@@ -15,7 +15,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.76 2000/07/29 03:26:42 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/adt/selfuncs.c,v 1.77 2000/08/03 00:58:22 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -252,9 +252,33 @@ eqsel(PG_FUNCTION_ARGS)
 Datum
 neqsel(PG_FUNCTION_ARGS)
 {
+	Oid			opid = PG_GETARG_OID(0);
+	Oid			relid = PG_GETARG_OID(1);
+	AttrNumber	attno = PG_GETARG_INT16(2);
+	Datum		value = PG_GETARG_DATUM(3);
+	int32		flag = PG_GETARG_INT32(4);
+	Oid			eqopid;
 	float8		result;
 
-	result = DatumGetFloat8(eqsel(fcinfo));
+	/*
+	 * We want 1 - eqsel() where the equality operator is the one associated
+	 * with this != operator, that is, its negator.
+	 */
+	eqopid = get_negator(opid);
+	if (eqopid)
+	{
+		result = DatumGetFloat8(DirectFunctionCall5(eqsel,
+													ObjectIdGetDatum(eqopid),
+													ObjectIdGetDatum(relid),
+													Int16GetDatum(attno),
+													value,
+													Int32GetDatum(flag)));
+	}
+	else
+	{
+		/* Use default selectivity (should we raise an error instead?) */
+		result = DEFAULT_EQ_SEL;
+	}
 	result = 1.0 - result;
 	PG_RETURN_FLOAT8(result);
 }
@@ -392,15 +416,39 @@ scalarltsel(PG_FUNCTION_ARGS)
 Datum
 scalargtsel(PG_FUNCTION_ARGS)
 {
+	Oid			opid = PG_GETARG_OID(0);
+	Oid			relid = PG_GETARG_OID(1);
+	AttrNumber	attno = PG_GETARG_INT16(2);
+	Datum		value = PG_GETARG_DATUM(3);
+	int32		flag = PG_GETARG_INT32(4);
+	Oid			ltopid;
 	float8		result;
 
 	/*
 	 * Compute selectivity of "<", then invert --- but only if we were
-	 * able to produce a non-default estimate.
+	 * able to produce a non-default estimate.  Note that we get the
+	 * negator which strictly speaking means we are looking at "<="
+	 * for ">" or "<" for ">=".  We assume this won't matter.
 	 */
-	result = DatumGetFloat8(scalarltsel(fcinfo));
+	ltopid = get_negator(opid);
+	if (ltopid)
+	{
+		result = DatumGetFloat8(DirectFunctionCall5(scalarltsel,
+													ObjectIdGetDatum(ltopid),
+													ObjectIdGetDatum(relid),
+													Int16GetDatum(attno),
+													value,
+													Int32GetDatum(flag)));
+	}
+	else
+	{
+		/* Use default selectivity (should we raise an error instead?) */
+		result = DEFAULT_INEQ_SEL;
+	}
+
 	if (result != DEFAULT_INEQ_SEL)
 		result = 1.0 - result;
+
 	PG_RETURN_FLOAT8(result);
 }
 
@@ -567,7 +615,7 @@ nlikesel(PG_FUNCTION_ARGS)
 Datum
 eqjoinsel(PG_FUNCTION_ARGS)
 {
-#ifdef NOT_USED
+#ifdef NOT_USED					/* see neqjoinsel() before removing me! */
 	Oid			opid = PG_GETARG_OID(0);
 #endif
 	Oid			relid1 = PG_GETARG_OID(1);
@@ -620,6 +668,11 @@ neqjoinsel(PG_FUNCTION_ARGS)
 {
 	float8		result;
 
+	/*
+	 * XXX we skip looking up the negator operator here because we know
+	 * eqjoinsel() won't look at it anyway.  If eqjoinsel() ever does look,
+	 * this routine will need to look more like neqsel() does.
+	 */
 	result = DatumGetFloat8(eqjoinsel(fcinfo));
 	result = 1.0 - result;
 	PG_RETURN_FLOAT8(result);