Avoid ERRCODE_INTERNAL_ERROR in oracle_compat.c functions.

repeat() checked for integer overflow during its calculation of the
required output space, but it just passed the resulting integer to
palloc().  This meant that result sizes between 1GB and 2GB led to
ERRCODE_INTERNAL_ERROR, "invalid memory alloc request size" rather
than ERRCODE_PROGRAM_LIMIT_EXCEEDED, "requested length too large".
That seems like a bit of a wart, so add an explicit AllocSizeIsValid
check to make these error cases uniform.

Do likewise in the sibling functions lpad() etc.  While we're here,
also modernize their overflow checks to use pg_mul_s32_overflow() etc
instead of expensive divisions.

Per complaint from Japin Li.  This is basically cosmetic, so I don't
feel a need to back-patch.

Discussion: https://postgr.es/m/ME3P282MB16676ED32167189CB0462173B6D69@ME3P282MB1667.AUSP282.PROD.OUTLOOK.COM

1 files changed, 24 insertions, 19 deletions

diff --git a/src/backend/utils/adt/oracle_compat.c b/src/backend/utils/adt/oracle_compat.c
index a6e043c32c5..018e8c93428 100644
--- a/src/backend/utils/adt/oracle_compat.c
+++ b/src/backend/utils/adt/oracle_compat.c
@@ -20,6 +20,8 @@
 #include "miscadmin.h"
 #include "utils/builtins.h"
 #include "utils/formatting.h"
+#include "utils/memutils.h"
+
 
 static text *dotrim(const char *string, int stringlen,
 					const char *set, int setlen,
@@ -155,7 +157,6 @@ lpad(PG_FUNCTION_ARGS)
 	int			m,
 				s1len,
 				s2len;
-
 	int			bytelen;
 
 	/* Negative len is silently taken as zero */
@@ -178,15 +179,16 @@ lpad(PG_FUNCTION_ARGS)
 	if (s2len <= 0)
 		len = s1len;			/* nothing to pad with, so don't pad */
 
-	bytelen = pg_database_encoding_max_length() * len;
-
-	/* check for integer overflow */
-	if (len != 0 && bytelen / pg_database_encoding_max_length() != len)
+	/* compute worst-case output length */
+	if (unlikely(pg_mul_s32_overflow(pg_database_encoding_max_length(), len,
+									 &bytelen)) ||
+		unlikely(pg_add_s32_overflow(bytelen, VARHDRSZ, &bytelen)) ||
+		unlikely(!AllocSizeIsValid(bytelen)))
 		ereport(ERROR,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("requested length too large")));
 
-	ret = (text *) palloc(VARHDRSZ + bytelen);
+	ret = (text *) palloc(bytelen);
 
 	m = len - s1len;
 
@@ -253,7 +255,6 @@ rpad(PG_FUNCTION_ARGS)
 	int			m,
 				s1len,
 				s2len;
-
 	int			bytelen;
 
 	/* Negative len is silently taken as zero */
@@ -276,15 +277,17 @@ rpad(PG_FUNCTION_ARGS)
 	if (s2len <= 0)
 		len = s1len;			/* nothing to pad with, so don't pad */
 
-	bytelen = pg_database_encoding_max_length() * len;
-
-	/* Check for integer overflow */
-	if (len != 0 && bytelen / pg_database_encoding_max_length() != len)
+	/* compute worst-case output length */
+	if (unlikely(pg_mul_s32_overflow(pg_database_encoding_max_length(), len,
+									 &bytelen)) ||
+		unlikely(pg_add_s32_overflow(bytelen, VARHDRSZ, &bytelen)) ||
+		unlikely(!AllocSizeIsValid(bytelen)))
 		ereport(ERROR,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("requested length too large")));
 
-	ret = (text *) palloc(VARHDRSZ + bytelen);
+	ret = (text *) palloc(bytelen);
+
 	m = len - s1len;
 
 	ptr1 = VARDATA_ANY(string1);
@@ -805,7 +808,7 @@ translate(PG_FUNCTION_ARGS)
 				tolen,
 				retlen,
 				i;
-	int			worst_len;
+	int			bytelen;
 	int			len;
 	int			source_len;
 	int			from_index;
@@ -824,15 +827,16 @@ translate(PG_FUNCTION_ARGS)
 	 * The worst-case expansion is to substitute a max-length character for a
 	 * single-byte character at each position of the string.
 	 */
-	worst_len = pg_database_encoding_max_length() * m;
-
-	/* check for integer overflow */
-	if (worst_len / pg_database_encoding_max_length() != m)
+	if (unlikely(pg_mul_s32_overflow(pg_database_encoding_max_length(), m,
+									 &bytelen)) ||
+		unlikely(pg_add_s32_overflow(bytelen, VARHDRSZ, &bytelen)) ||
+		unlikely(!AllocSizeIsValid(bytelen)))
 		ereport(ERROR,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("requested length too large")));
 
-	result = (text *) palloc(worst_len + VARHDRSZ);
+	result = (text *) palloc(bytelen);
+
 	target = VARDATA(result);
 	retlen = 0;
 
@@ -1128,7 +1132,8 @@ repeat(PG_FUNCTION_ARGS)
 	slen = VARSIZE_ANY_EXHDR(string);
 
 	if (unlikely(pg_mul_s32_overflow(count, slen, &tlen)) ||
-		unlikely(pg_add_s32_overflow(tlen, VARHDRSZ, &tlen)))
+		unlikely(pg_add_s32_overflow(tlen, VARHDRSZ, &tlen)) ||
+		unlikely(!AllocSizeIsValid(tlen)))
 		ereport(ERROR,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("requested length too large")));