From 43a899f41f46918a0bf442edb091b08c214c68f8 Mon Sep 17 00:00:00 2001 From: Dean Rasheed Date: Sun, 1 Mar 2020 14:49:25 +0000 Subject: Fix corner-case loss of precision in numeric ln(). When deciding on the local rscale to use for the Taylor series expansion, ln_var() neglected to account for the fact that the result is subsequently multiplied by a factor of 2^(nsqrt+1), where nsqrt is the number of square root operations performed in the range reduction step, which can be as high as 22 for very large inputs. This could result in a loss of precision, particularly when combined with large rscale values, for which a large number of Taylor series terms is required (up to around 400). Fix by computing a few extra digits in the Taylor series, based on the weight of the multiplicative factor log10(2^(nsqrt+1)). It remains to be proven whether or not the other 8 extra digits used for the Taylor series is appropriate, but this at least deals with the obvious oversight of failing to account for the effects of the final multiplication. Per report from Justin AnyhowStep. Reviewed by Tom Lane. Discussion: https://postgr.es/m/16280-279f299d9c06e56f@postgresql.org --- src/backend/utils/adt/numeric.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/backend/utils/adt/numeric.c') diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c index bd00f23b946..10229eb37e9 100644 --- a/src/backend/utils/adt/numeric.c +++ b/src/backend/utils/adt/numeric.c @@ -8500,6 +8500,7 @@ ln_var(const NumericVar *arg, NumericVar *result, int rscale) NumericVar ni; NumericVar elem; NumericVar fact; + int nsqrt; int local_rscale; int cmp; @@ -8530,12 +8531,14 @@ ln_var(const NumericVar *arg, NumericVar *result, int rscale) * rscale as we work so that we keep this many significant digits at each * step (plus a few more for good measure). */ + nsqrt = 0; while (cmp_var(&x, &const_zero_point_nine) <= 0) { local_rscale = rscale - x.weight * DEC_DIGITS / 2 + 8; local_rscale = Max(local_rscale, NUMERIC_MIN_DISPLAY_SCALE); sqrt_var(&x, &x, local_rscale); mul_var(&fact, &const_two, &fact, 0); + nsqrt++; } while (cmp_var(&x, &const_one_point_one) >= 0) { @@ -8543,6 +8546,7 @@ ln_var(const NumericVar *arg, NumericVar *result, int rscale) local_rscale = Max(local_rscale, NUMERIC_MIN_DISPLAY_SCALE); sqrt_var(&x, &x, local_rscale); mul_var(&fact, &const_two, &fact, 0); + nsqrt++; } /* @@ -8555,8 +8559,12 @@ ln_var(const NumericVar *arg, NumericVar *result, int rscale) * * The convergence of this is not as fast as one would like, but is * tolerable given that z is small. + * + * The Taylor series result will be multiplied by 2^(nsqrt+1), which has a + * decimal weight of (nsqrt+1) * log10(2), so work with this many extra + * digits of precision (plus a few more for good measure). */ - local_rscale = rscale + 8; + local_rscale = rscale + (int) ((nsqrt + 1) * 0.301029995663981) + 8; sub_var(&x, &const_one, result); add_var(&x, &const_one, &elem); -- cgit v1.2.3