Improve error handling of cryptohash computations

The existing cryptohash facility was causing problems in some code paths
related to MD5 (frontend and backend) that relied on the fact that the
only type of error that could happen would be an OOM, as the MD5
implementation used in PostgreSQL ~13 (the in-core implementation is
used when compiling with or without OpenSSL in those older versions),
could fail only under this circumstance.

The new cryptohash facilities can fail for reasons other than OOMs, like
attempting MD5 when FIPS is enabled (upstream OpenSSL allows that up to
1.0.2, Fedora and Photon patch OpenSSL 1.1.1 to allow that), so this
would cause incorrect reports to show up.

This commit extends the cryptohash APIs so as callers of those routines
can fetch more context when an error happens, by using a new routine
called pg_cryptohash_error().  The error states are stored within each
implementation's internal context data, so as it is possible to extend
the logic depending on what's suited for an implementation.  The default
implementation requires few error states, but OpenSSL could report
various issues depending on its internal state so more is needed in
cryptohash_openssl.c, and the code is shaped so as we are always able to
grab the necessary information.

The core code is changed to adapt to the new error routine, painting
more "const" across the call stack where the static errors are stored,
particularly in authentication code paths on variables that provide
log details.  This way, any future changes would warn if attempting to
free these strings.  The MD5 authentication code was also a bit blurry
about the handling of "logdetail" (LOG sent to the postmaster), so
improve the comments related that, while on it.

The origin of the problem is 87ae969, that introduced the centralized
cryptohash facility.  Extra changes are done for pgcrypto in v14 for the
non-OpenSSL code path to cope with the improvements done by this
commit.

Reported-by: Michael Mühlbeyer
Author: Michael Paquier
Reviewed-by: Tom Lane
Discussion: https://postgr.es/m/89B7F072-5BBE-4C92-903E-D83E865D9367@trivadis.com
Backpatch-through: 14

1 files changed, 56 insertions, 1 deletions

diff --git a/src/common/cryptohash.c b/src/common/cryptohash.c
index b13815dd98e..ddf35d28b22 100644
--- a/src/common/cryptohash.c
+++ b/src/common/cryptohash.c
@@ -40,10 +40,18 @@
 #define FREE(ptr) free(ptr)
 #endif
 
+/* Set of error states */
+typedef enum pg_cryptohash_errno
+{
+	PG_CRYPTOHASH_ERROR_NONE = 0,
+	PG_CRYPTOHASH_ERROR_DEST_LEN
+} pg_cryptohash_errno;
+
 /* Internal pg_cryptohash_ctx structure */
 struct pg_cryptohash_ctx
 {
 	pg_cryptohash_type type;
+	pg_cryptohash_errno error;
 
 	union
 	{
@@ -76,9 +84,10 @@ pg_cryptohash_create(pg_cryptohash_type type)
 	ctx = ALLOC(sizeof(pg_cryptohash_ctx));
 	if (ctx == NULL)
 		return NULL;
+
 	memset(ctx, 0, sizeof(pg_cryptohash_ctx));
 	ctx->type = type;
-
+	ctx->error = PG_CRYPTOHASH_ERROR_NONE;
 	return ctx;
 }
 
@@ -174,32 +183,50 @@ pg_cryptohash_final(pg_cryptohash_ctx *ctx, uint8 *dest, size_t len)
 	{
 		case PG_MD5:
 			if (len < MD5_DIGEST_LENGTH)
+			{
+				ctx->error = PG_CRYPTOHASH_ERROR_DEST_LEN;
 				return -1;
+			}
 			pg_md5_final(&ctx->data.md5, dest);
 			break;
 		case PG_SHA1:
 			if (len < SHA1_DIGEST_LENGTH)
+			{
+				ctx->error = PG_CRYPTOHASH_ERROR_DEST_LEN;
 				return -1;
+			}
 			pg_sha1_final(&ctx->data.sha1, dest);
 			break;
 		case PG_SHA224:
 			if (len < PG_SHA224_DIGEST_LENGTH)
+			{
+				ctx->error = PG_CRYPTOHASH_ERROR_DEST_LEN;
 				return -1;
+			}
 			pg_sha224_final(&ctx->data.sha224, dest);
 			break;
 		case PG_SHA256:
 			if (len < PG_SHA256_DIGEST_LENGTH)
+			{
+				ctx->error = PG_CRYPTOHASH_ERROR_DEST_LEN;
 				return -1;
+			}
 			pg_sha256_final(&ctx->data.sha256, dest);
 			break;
 		case PG_SHA384:
 			if (len < PG_SHA384_DIGEST_LENGTH)
+			{
+				ctx->error = PG_CRYPTOHASH_ERROR_DEST_LEN;
 				return -1;
+			}
 			pg_sha384_final(&ctx->data.sha384, dest);
 			break;
 		case PG_SHA512:
 			if (len < PG_SHA512_DIGEST_LENGTH)
+			{
+				ctx->error = PG_CRYPTOHASH_ERROR_DEST_LEN;
 				return -1;
+			}
 			pg_sha512_final(&ctx->data.sha512, dest);
 			break;
 	}
@@ -221,3 +248,31 @@ pg_cryptohash_free(pg_cryptohash_ctx *ctx)
 	explicit_bzero(ctx, sizeof(pg_cryptohash_ctx));
 	FREE(ctx);
 }
+
+/*
+ * pg_cryptohash_error
+ *
+ * Returns a static string providing errors about an error that
+ * happened during a computation.
+ */
+const char *
+pg_cryptohash_error(pg_cryptohash_ctx *ctx)
+{
+	/*
+	 * This implementation would never fail because of an out-of-memory error,
+	 * except when creating the context.
+	 */
+	if (ctx == NULL)
+		return _("out of memory");
+
+	switch (ctx->error)
+	{
+		case PG_CRYPTOHASH_ERROR_NONE:
+			return _("success");
+		case PG_CRYPTOHASH_ERROR_DEST_LEN:
+			return _("destination buffer too small");
+	}
+
+	Assert(false);
+	return _("success");
+}