pgstat: store statistics in shared memory.

Previously the statistics collector received statistics updates via UDP and
shared statistics data by writing them out to temporary files regularly. These
files can reach tens of megabytes and are written out up to twice a
second. This has repeatedly prevented us from adding additional useful
statistics.

Now statistics are stored in shared memory. Statistics for variable-numbered
objects are stored in a dshash hashtable (backed by dynamic shared
memory). Fixed-numbered stats are stored in plain shared memory.

The header for pgstat.c contains an overview of the architecture.

The stats collector is not needed anymore, remove it.

By utilizing the transactional statistics drop infrastructure introduced in a
prior commit statistics entries cannot "leak" anymore. Previously leaked
statistics were dropped by pgstat_vacuum_stat(), called from [auto-]vacuum. On
systems with many small relations pgstat_vacuum_stat() could be quite
expensive.

Now that replicas drop statistics entries for dropped objects, it is not
necessary anymore to reset stats when starting from a cleanly shut down
replica.

Subsequent commits will perform some further code cleanup, adapt docs and add
tests.

Bumps PGSTAT_FILE_FORMAT_ID.

Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-By: Andres Freund <andres@anarazel.de>
Reviewed-By: Thomas Munro <thomas.munro@gmail.com>
Reviewed-By: Justin Pryzby <pryzby@telsasoft.com>
Reviewed-By: "David G. Johnston" <david.g.johnston@gmail.com>
Reviewed-By: Tomas Vondra <tomas.vondra@2ndquadrant.com> (in a much earlier version)
Reviewed-By: Arthur Zakirov <a.zakirov@postgrespro.ru> (in a much earlier version)
Reviewed-By: Antonin Houska <ah@cybertec.at> (in a much earlier version)
Discussion: https://postgr.es/m/20220303021600.hs34ghqcw6zcokdh@alap3.anarazel.de
Discussion: https://postgr.es/m/20220308205351.2xcn6k4x5yivcxyd@alap3.anarazel.de
Discussion: https://postgr.es/m/20210319235115.y3wz7hpnnrshdyv6@alap3.anarazel.de

1 files changed, 82 insertions, 85 deletions

diff --git a/src/backend/utils/activity/pgstat_function.c b/src/backend/utils/activity/pgstat_function.c
index ad9879afb2a..427d8c47fc6 100644
--- a/src/backend/utils/activity/pgstat_function.c
+++ b/src/backend/utils/activity/pgstat_function.c
@@ -17,8 +17,10 @@
 
 #include "postgres.h"
 
+#include "fmgr.h"
+#include "utils/inval.h"
 #include "utils/pgstat_internal.h"
-#include "utils/timestamp.h"
+#include "utils/syscache.h"
 
 
 /* ----------
@@ -29,18 +31,6 @@ int			pgstat_track_functions = TRACK_FUNC_OFF;
 
 
 /*
- * Indicates if backend has some function stats that it hasn't yet
- * sent to the collector.
- */
-bool		have_function_stats = false;
-
-/*
- * Backends store per-function info that's waiting to be sent to the collector
- * in this hash table (indexed by function OID).
- */
-static HTAB *pgStatFunctions = NULL;
-
-/*
  * Total time charged to functions so far in the current backend.
  * We use this to help separate "self" and "other" time charges.
  * (We assume this initializes to zero.)
@@ -61,6 +51,10 @@ pgstat_create_function(Oid proid)
 
 /*
  * Ensure that stats are dropped if transaction commits.
+ *
+ * NB: This is only reliable because pgstat_init_function_usage() does some
+ * extra work. If other places start emitting function stats they likely need
+ * similar logic.
  */
 void
 pgstat_drop_function(Oid proid)
@@ -78,8 +72,9 @@ void
 pgstat_init_function_usage(FunctionCallInfo fcinfo,
 						   PgStat_FunctionCallUsage *fcu)
 {
-	PgStat_BackendFunctionEntry *htabent;
-	bool		found;
+	PgStat_EntryRef *entry_ref;
+	PgStat_BackendFunctionEntry *pending;
+	bool		created_entry;
 
 	if (pgstat_track_functions <= fcinfo->flinfo->fn_stats)
 	{
@@ -88,29 +83,48 @@ pgstat_init_function_usage(FunctionCallInfo fcinfo,
 		return;
 	}
 
-	if (!pgStatFunctions)
+	entry_ref = pgstat_prep_pending_entry(PGSTAT_KIND_FUNCTION,
+										  MyDatabaseId,
+										  fcinfo->flinfo->fn_oid,
+										  &created_entry);
+
+	/*
+	 * If no shared entry already exists, check if the function has been
+	 * deleted concurrently. This can go unnoticed until here because
+	 * executing a statement that just calls a function, does not trigger
+	 * cache invalidation processing. The reason we care about this case is
+	 * that otherwise we could create a new stats entry for an already dropped
+	 * function (for relations etc this is not possible because emitting stats
+	 * requires a lock for the relation to already have been acquired).
+	 *
+	 * It's somewhat ugly to have a behavioral difference based on
+	 * track_functions being enabled/disabled. But it seems acceptable, given
+	 * that there's already behavioral differences depending on whether the
+	 * function is the caches etc.
+	 *
+	 * For correctness it'd be sufficient to set ->dropped to true. However,
+	 * the accepted invalidation will commonly cause "low level" failures in
+	 * PL code, with an OID in the error message. Making this harder to
+	 * test...
+	 */
+	if (created_entry)
 	{
-		/* First time through - initialize function stat table */
-		HASHCTL		hash_ctl;
-
-		hash_ctl.keysize = sizeof(Oid);
-		hash_ctl.entrysize = sizeof(PgStat_BackendFunctionEntry);
-		pgStatFunctions = hash_create("Function stat entries",
-									  PGSTAT_FUNCTION_HASH_SIZE,
-									  &hash_ctl,
-									  HASH_ELEM | HASH_BLOBS);
+		AcceptInvalidationMessages();
+		if (!SearchSysCacheExists1(PROCOID, ObjectIdGetDatum(fcinfo->flinfo->fn_oid)))
+		{
+			pgstat_drop_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId,
+							  fcinfo->flinfo->fn_oid);
+			ereport(ERROR, errcode(ERRCODE_UNDEFINED_FUNCTION),
+					errmsg("function call to dropped function"));
+		}
 	}
 
-	/* Get the stats entry for this function, create if necessary */
-	htabent = hash_search(pgStatFunctions, &fcinfo->flinfo->fn_oid,
-						  HASH_ENTER, &found);
-	if (!found)
-		MemSet(&htabent->f_counts, 0, sizeof(PgStat_FunctionCounts));
+	pending = entry_ref->pending;
 
-	fcu->fs = &htabent->f_counts;
+	fcu->fs = &pending->f_counts;
 
 	/* save stats for this function, later used to compensate for recursion */
-	fcu->save_f_total_time = htabent->f_counts.f_total_time;
+	fcu->save_f_total_time = pending->f_counts.f_total_time;
 
 	/* save current backend-wide total time */
 	fcu->save_total = total_func_time;
@@ -167,64 +181,37 @@ pgstat_end_function_usage(PgStat_FunctionCallUsage *fcu, bool finalize)
 		fs->f_numcalls++;
 	fs->f_total_time = f_total;
 	INSTR_TIME_ADD(fs->f_self_time, f_self);
-
-	/* indicate that we have something to send */
-	have_function_stats = true;
 }
 
 /*
- * Subroutine for pgstat_report_stat: populate and send a function stat message
+ * Flush out pending stats for the entry
+ *
+ * If nowait is true, this function returns false if lock could not
+ * immediately acquired, otherwise true is returned.
  */
-void
-pgstat_send_funcstats(void)
+bool
+pgstat_function_flush_cb(PgStat_EntryRef *entry_ref, bool nowait)
 {
-	/* we assume this inits to all zeroes: */
-	static const PgStat_FunctionCounts all_zeroes;
+	PgStat_BackendFunctionEntry *localent;
+	PgStatShared_Function *shfuncent;
 
-	PgStat_MsgFuncstat msg;
-	PgStat_BackendFunctionEntry *entry;
-	HASH_SEQ_STATUS fstat;
+	localent = (PgStat_BackendFunctionEntry *) entry_ref->pending;
+	shfuncent = (PgStatShared_Function *) entry_ref->shared_stats;
 
-	if (pgStatFunctions == NULL)
-		return;
-
-	pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_FUNCSTAT);
-	msg.m_databaseid = MyDatabaseId;
-	msg.m_nentries = 0;
-
-	hash_seq_init(&fstat, pgStatFunctions);
-	while ((entry = (PgStat_BackendFunctionEntry *) hash_seq_search(&fstat)) != NULL)
-	{
-		PgStat_FunctionEntry *m_ent;
+	/* localent always has non-zero content */
 
-		/* Skip it if no counts accumulated since last time */
-		if (memcmp(&entry->f_counts, &all_zeroes,
-				   sizeof(PgStat_FunctionCounts)) == 0)
-			continue;
-
-		/* need to convert format of time accumulators */
-		m_ent = &msg.m_entry[msg.m_nentries];
-		m_ent->f_id = entry->f_id;
-		m_ent->f_numcalls = entry->f_counts.f_numcalls;
-		m_ent->f_total_time = INSTR_TIME_GET_MICROSEC(entry->f_counts.f_total_time);
-		m_ent->f_self_time = INSTR_TIME_GET_MICROSEC(entry->f_counts.f_self_time);
-
-		if (++msg.m_nentries >= PGSTAT_NUM_FUNCENTRIES)
-		{
-			pgstat_send(&msg, offsetof(PgStat_MsgFuncstat, m_entry[0]) +
-						msg.m_nentries * sizeof(PgStat_FunctionEntry));
-			msg.m_nentries = 0;
-		}
+	if (!pgstat_lock_entry(entry_ref, nowait))
+		return false;
 
-		/* reset the entry's counts */
-		MemSet(&entry->f_counts, 0, sizeof(PgStat_FunctionCounts));
-	}
+	shfuncent->stats.f_numcalls += localent->f_counts.f_numcalls;
+	shfuncent->stats.f_total_time +=
+		INSTR_TIME_GET_MICROSEC(localent->f_counts.f_total_time);
+	shfuncent->stats.f_self_time +=
+		INSTR_TIME_GET_MICROSEC(localent->f_counts.f_self_time);
 
-	if (msg.m_nentries > 0)
-		pgstat_send(&msg, offsetof(PgStat_MsgFuncstat, m_entry[0]) +
-					msg.m_nentries * sizeof(PgStat_FunctionEntry));
+	pgstat_unlock_entry(entry_ref);
 
-	have_function_stats = false;
+	return true;
 }
 
 /*
@@ -235,12 +222,22 @@ pgstat_send_funcstats(void)
 PgStat_BackendFunctionEntry *
 find_funcstat_entry(Oid func_id)
 {
-	pgstat_assert_is_up();
+	PgStat_EntryRef *entry_ref;
 
-	if (pgStatFunctions == NULL)
-		return NULL;
+	entry_ref = pgstat_fetch_pending_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId, func_id);
+
+	if (entry_ref)
+		return entry_ref->pending;
+	return NULL;
+}
 
-	return (PgStat_BackendFunctionEntry *) hash_search(pgStatFunctions,
-													   (void *) &func_id,
-													   HASH_FIND, NULL);
+/*
+ * Support function for the SQL-callable pgstat* functions. Returns
+ * the collected statistics for one function or NULL.
+ */
+PgStat_StatFuncEntry *
+pgstat_fetch_stat_funcentry(Oid func_id)
+{
+	return (PgStat_StatFuncEntry *)
+		pgstat_fetch_entry(PGSTAT_KIND_FUNCTION, MyDatabaseId, func_id);
 }