Add support for file inclusions in HBA and ident configuration files

pg_hba.conf and pg_ident.conf gain support for three record keywords:
- "include", to include a file.
- "include_if_exists", to include a file, ignoring it if missing.
- "include_dir", to include a directory of files.  These are classified
by name (C locale, mostly) and need to be prefixed by ".conf", hence
following the same rules as GUCs.

This commit relies on the refactoring pieces done in efc9816, ad6c528,
783e8c6 and 1b73d0b, adding a small wrapper to build a list of
TokenizedAuthLines (tokenize_include_file), and the code is shaped to
offer some symmetry with what is done for GUCs with the same options.

pg_hba_file_rules and pg_ident_file_mappings gain a new field called
file_name, to track from which file a record is located, taking
advantage of the addition of rule_number in c591300 to offer an
organized view of the HBA or ident records loaded.

Bump catalog version.

Author: Julien Rouhaud
Reviewed-by: Michael Paquier
Discussion: https://postgr.es/m/20220223045959.35ipdsvbxcstrhya@jrouhaud

7 files changed, 260 insertions, 44 deletions

diff --git a/src/backend/libpq/hba.c b/src/backend/libpq/hba.c
index 1bd151c9416..862ec18e913 100644
--- a/src/backend/libpq/hba.c
+++ b/src/backend/libpq/hba.c
@@ -464,15 +464,73 @@ next_field_expand(const char *filename, char **lineptr,
 }
 
 /*
+ * tokenize_include_file
+ *		Include a file from another file into an hba "field".
+ *
+ * Opens and tokenises a file included from another authentication file
+ * with one of the include records ("include", "include_if_exists" or
+ * "include_dir"), and assign all values found to an existing list of
+ * list of AuthTokens.
+ *
+ * All new tokens are allocated in the memory context dedicated to the
+ * tokenization, aka tokenize_context.
+ *
+ * If missing_ok is true, ignore a missing file.
+ *
+ * In event of an error, log a message at ereport level elevel, and also
+ * set *err_msg to a string describing the error.  Note that the result
+ * may be non-NIL anyway, so *err_msg must be tested to determine whether
+ * there was an error.
+ */
+static void
+tokenize_include_file(const char *outer_filename,
+					  const char *inc_filename,
+					  List **tok_lines,
+					  int elevel,
+					  int depth,
+					  bool missing_ok,
+					  char **err_msg)
+{
+	char	   *inc_fullname;
+	FILE	   *inc_file;
+
+	inc_fullname = AbsoluteConfigLocation(inc_filename, outer_filename);
+	inc_file = open_auth_file(inc_fullname, elevel, depth, err_msg);
+
+	if (!inc_file)
+	{
+		if (errno == ENOENT && missing_ok)
+		{
+			ereport(elevel,
+					(errmsg("skipping missing authentication file \"%s\"",
+							inc_fullname)));
+			*err_msg = NULL;
+			pfree(inc_fullname);
+			return;
+		}
+
+		/* error in err_msg, so leave and report */
+		pfree(inc_fullname);
+		Assert(err_msg);
+		return;
+	}
+
+	tokenize_auth_file(inc_fullname, inc_file, tok_lines, elevel,
+					   depth);
+	free_auth_file(inc_file, depth);
+	pfree(inc_fullname);
+}
+
+/*
  * tokenize_expand_file
  *		Expand a file included from another file into an hba "field"
  *
  * Opens and tokenises a file included from another HBA config file with @,
  * and returns all values found therein as a flat list of AuthTokens.  If a
- * @-token is found, recursively expand it.  The newly read tokens are
- * appended to "tokens" (so that foo,bar,@baz does what you expect).
- * All new tokens are allocated in the memory context dedicated to the
- * list of TokenizedAuthLines, aka tokenize_context.
+ * @-token or include record is found, recursively expand it.  The newly
+ * read tokens are appended to "tokens" (so that foo,bar,@baz does what you
+ * expect).  All new tokens are allocated in the memory context dedicated
+ * to the list of TokenizedAuthLines, aka tokenize_context.
  *
  * In event of an error, log a message at ereport level elevel, and also
  * set *err_msg to a string describing the error.  Note that the result
@@ -502,7 +560,10 @@ tokenize_expand_file(List *tokens,
 		return tokens;
 	}
 
-	/* There is possible recursion here if the file contains @ */
+	/*
+	 * There is possible recursion here if the file contains @ or an include
+	 * records.
+	 */
 	tokenize_auth_file(inc_fullname, inc_file, &inc_lines, elevel,
 					   depth);
 
@@ -706,6 +767,8 @@ tokenize_auth_file(const char *filename, FILE *file, List **tok_lines,
 
 	while (!feof(file) && !ferror(file))
 	{
+		TokenizedAuthLine *tok_line;
+		MemoryContext oldcxt;
 		char	   *lineptr;
 		List	   *current_line = NIL;
 		char	   *err_msg = NULL;
@@ -763,8 +826,6 @@ tokenize_auth_file(const char *filename, FILE *file, List **tok_lines,
 			/* add field to line, unless we are at EOL or comment start */
 			if (current_field != NIL)
 			{
-				MemoryContext oldcxt;
-
 				/*
 				 * lappend() may do its own allocations, so move to the
 				 * context for the list of tokens.
@@ -776,24 +837,115 @@ tokenize_auth_file(const char *filename, FILE *file, List **tok_lines,
 		}
 
 		/*
-		 * Reached EOL; emit line to TokenizedAuthLine list unless it's boring
+		 * Reached EOL; no need to emit line to TokenizedAuthLine list if it's
+		 * boring.
 		 */
-		if (current_line != NIL || err_msg != NULL)
+		if (current_line == NIL && err_msg == NULL)
+			goto next_line;
+
+		/* If the line is valid, check if that's an include directive */
+		if (err_msg == NULL && list_length(current_line) == 2)
 		{
-			TokenizedAuthLine *tok_line;
-			MemoryContext oldcxt;
+			AuthToken  *first,
+					   *second;
 
-			oldcxt = MemoryContextSwitchTo(tokenize_context);
-			tok_line = (TokenizedAuthLine *) palloc(sizeof(TokenizedAuthLine));
-			tok_line->fields = current_line;
-			tok_line->file_name = pstrdup(filename);
-			tok_line->line_num = line_number;
-			tok_line->raw_line = pstrdup(buf.data);
-			tok_line->err_msg = err_msg ? pstrdup(err_msg) : NULL;
-			*tok_lines = lappend(*tok_lines, tok_line);
-			MemoryContextSwitchTo(oldcxt);
+			first = linitial(linitial_node(List, current_line));
+			second = linitial(lsecond_node(List, current_line));
+
+			if (strcmp(first->string, "include") == 0)
+			{
+				tokenize_include_file(filename, second->string, tok_lines,
+									  elevel, depth + 1, false, &err_msg);
+
+				if (err_msg)
+					goto process_line;
+
+				/*
+				 * tokenize_auth_file() has taken care of creating the
+				 * TokenizedAuthLines.
+				 */
+				goto next_line;
+			}
+			else if (strcmp(first->string, "include_dir") == 0)
+			{
+				char	  **filenames;
+				char	   *dir_name = second->string;
+				int			num_filenames;
+				StringInfoData err_buf;
+
+				filenames = GetConfFilesInDir(dir_name, filename, elevel,
+											  &num_filenames, &err_msg);
+
+				if (!filenames)
+				{
+					/* the error is in err_msg, so create an entry */
+					goto process_line;
+				}
+
+				initStringInfo(&err_buf);
+				for (int i = 0; i < num_filenames; i++)
+				{
+					tokenize_include_file(filename, filenames[i], tok_lines,
+										  elevel, depth + 1, false, &err_msg);
+					/* cumulate errors if any */
+					if (err_msg)
+					{
+						if (err_buf.len > 0)
+							appendStringInfoChar(&err_buf, '\n');
+						appendStringInfoString(&err_buf, err_msg);
+					}
+				}
+
+				/* clean up things */
+				for (int i = 0; i < num_filenames; i++)
+					pfree(filenames[i]);
+				pfree(filenames);
+
+				/*
+				 * If there were no errors, the line is fully processed,
+				 * bypass the general TokenizedAuthLine processing.
+				 */
+				if (err_buf.len == 0)
+					goto next_line;
+
+				/* Otherwise, process the cumulated errors, if any. */
+				err_msg = err_buf.data;
+				goto process_line;
+			}
+			else if (strcmp(first->string, "include_if_exists") == 0)
+			{
+
+				tokenize_include_file(filename, second->string, tok_lines,
+									  elevel, depth + 1, true, &err_msg);
+				if (err_msg)
+					goto process_line;
+
+				/*
+				 * tokenize_auth_file() has taken care of creating the
+				 * TokenizedAuthLines.
+				 */
+				goto next_line;
+			}
 		}
 
+process_line:
+
+		/*
+		 * General processing: report the error if any and emit line to the
+		 * TokenizedAuthLine.  This is saved in the memory context dedicated
+		 * to this list.
+		 */
+		oldcxt = MemoryContextSwitchTo(tokenize_context);
+		tok_line = (TokenizedAuthLine *) palloc0(sizeof(TokenizedAuthLine));
+		tok_line->fields = current_line;
+		tok_line->file_name = pstrdup(filename);
+		tok_line->line_num = line_number;
+		tok_line->raw_line = pstrdup(buf.data);
+		tok_line->err_msg = err_msg ? pstrdup(err_msg) : NULL;
+		*tok_lines = lappend(*tok_lines, tok_line);
+		MemoryContextSwitchTo(oldcxt);
+
+next_line:
 		line_number += continuations + 1;
 		callback_arg.linenum = line_number;
 	}
diff --git a/src/backend/libpq/pg_hba.conf.sample b/src/backend/libpq/pg_hba.conf.sample
index 5f3f63eb0c5..095e3b4cc00 100644
--- a/src/backend/libpq/pg_hba.conf.sample
+++ b/src/backend/libpq/pg_hba.conf.sample
@@ -5,6 +5,10 @@
 # documentation for a complete description of this file.  A short
 # synopsis follows.
 #
+# ----------------------
+# Authentication Records
+# ----------------------
+#
 # This file controls: which hosts are allowed to connect, how clients
 # are authenticated, which PostgreSQL user names they can use, which
 # databases they can access.  Records take one of these forms:
@@ -64,11 +68,34 @@
 # its special character, and just match a database or username with
 # that name.
 #
+# ---------------
+# Include Records
+# ---------------
+#
+# This file allows the inclusion of external files or directories holding
+# more records, using the following keywords:
+#
+# include           FILE
+# include_if_exists FILE
+# include_dir       DIRECTORY
+#
+# FILE is the file name to include, and DIR is the directory name containing
+# the file(s) to include.  Any file in a directory will be loaded if suffixed
+# with ".conf".  The files of a directory are ordered by name.
+# include_if_exists ignores missing files.  FILE and DIRECTORY can be
+# specified as a relative or an absolute path, and can be double-quoted if
+# they contain spaces.
+#
+# -------------
+# Miscellaneous
+# -------------
+#
 # This file is read on server startup and when the server receives a
 # SIGHUP signal.  If you edit the file on a running system, you have to
 # SIGHUP the server for the changes to take effect, run "pg_ctl reload",
 # or execute "SELECT pg_reload_conf()".
 #
+# ----------------------------------
 # Put your actual configuration here
 # ----------------------------------
 #
diff --git a/src/backend/libpq/pg_ident.conf.sample b/src/backend/libpq/pg_ident.conf.sample
index a5870e6448c..5d32684b28f 100644
--- a/src/backend/libpq/pg_ident.conf.sample
+++ b/src/backend/libpq/pg_ident.conf.sample
@@ -1,6 +1,10 @@
 # PostgreSQL User Name Maps
 # =========================
 #
+# ---------------
+# Mapping Records
+# ---------------
+#
 # Refer to the PostgreSQL documentation, chapter "Client
 # Authentication" for a complete description.  A short synopsis
 # follows.
@@ -31,6 +35,28 @@
 # system user names and PostgreSQL user names are the same, you don't
 # need anything in this file.
 #
+# ---------------
+# Include Records
+# ---------------
+#
+# This file allows the inclusion of external files or directories holding
+# more records, using the following keywords:
+#
+# include           FILE
+# include_if_exists FILE
+# include_dir       DIRECTORY
+#
+# FILE is the file name to include, and DIR is the directory name containing
+# the file(s) to include.  Any file in a directory will be loaded if suffixed
+# with ".conf".  The files of a directory are ordered by name.
+# include_if_exists ignores missing files.  FILE and DIRECTORY can be
+# specified as a relative or an absolute path, and can be double-quoted if
+# they contain spaces.
+#
+# -------------------------------
+# Miscellaneous
+# -------------------------------
+#
 # This file is read on server startup and when the postmaster receives
 # a SIGHUP signal.  If you edit the file on a running system, you have
 # to SIGHUP the postmaster for the changes to take effect.  You can
diff --git a/src/backend/utils/adt/hbafuncs.c b/src/backend/utils/adt/hbafuncs.c
index 87996da487b..633eda30d39 100644
--- a/src/backend/utils/adt/hbafuncs.c
+++ b/src/backend/utils/adt/hbafuncs.c
@@ -26,12 +26,12 @@
 
 static ArrayType *get_hba_options(HbaLine *hba);
 static void fill_hba_line(Tuplestorestate *tuple_store, TupleDesc tupdesc,
-						  int rule_number, int lineno, HbaLine *hba,
-						  const char *err_msg);
+						  int rule_number, char *filename, int lineno,
+						  HbaLine *hba, const char *err_msg);
 static void fill_hba_view(Tuplestorestate *tuple_store, TupleDesc tupdesc);
 static void fill_ident_line(Tuplestorestate *tuple_store, TupleDesc tupdesc,
-							int map_number, int lineno, IdentLine *ident,
-							const char *err_msg);
+							int map_number, char *filename, int lineno,
+							IdentLine *ident, const char *err_msg);
 static void fill_ident_view(Tuplestorestate *tuple_store, TupleDesc tupdesc);
 
 
@@ -159,7 +159,7 @@ get_hba_options(HbaLine *hba)
 }
 
 /* Number of columns in pg_hba_file_rules view */
-#define NUM_PG_HBA_FILE_RULES_ATTS	 10
+#define NUM_PG_HBA_FILE_RULES_ATTS	 11
 
 /*
  * fill_hba_line
@@ -168,7 +168,8 @@ get_hba_options(HbaLine *hba)
  * tuple_store: where to store data
  * tupdesc: tuple descriptor for the view
  * rule_number: unique identifier among all valid rules
- * lineno: pg_hba.conf line number (must always be valid)
+ * filename: configuration file name (must always be valid)
+ * lineno: line number of configuration file (must always be valid)
  * hba: parsed line data (can be NULL, in which case err_msg should be set)
  * err_msg: error message (NULL if none)
  *
@@ -177,7 +178,7 @@ get_hba_options(HbaLine *hba)
  */
 static void
 fill_hba_line(Tuplestorestate *tuple_store, TupleDesc tupdesc,
-			  int rule_number, int lineno, HbaLine *hba,
+			  int rule_number, char *filename, int lineno, HbaLine *hba,
 			  const char *err_msg)
 {
 	Datum		values[NUM_PG_HBA_FILE_RULES_ATTS];
@@ -203,6 +204,9 @@ fill_hba_line(Tuplestorestate *tuple_store, TupleDesc tupdesc,
 	else
 		values[index++] = Int32GetDatum(rule_number);
 
+	/* file_name */
+	values[index++] = CStringGetTextDatum(filename);
+
 	/* line_number */
 	values[index++] = Int32GetDatum(lineno);
 
@@ -346,7 +350,7 @@ fill_hba_line(Tuplestorestate *tuple_store, TupleDesc tupdesc,
 	else
 	{
 		/* no parsing result, so set relevant fields to nulls */
-		memset(&nulls[2], true, (NUM_PG_HBA_FILE_RULES_ATTS - 3) * sizeof(bool));
+		memset(&nulls[3], true, (NUM_PG_HBA_FILE_RULES_ATTS - 4) * sizeof(bool));
 	}
 
 	/* error */
@@ -402,7 +406,8 @@ fill_hba_view(Tuplestorestate *tuple_store, TupleDesc tupdesc)
 			rule_number++;
 
 		fill_hba_line(tuple_store, tupdesc, rule_number,
-					  tok_line->line_num, hbaline, tok_line->err_msg);
+					  tok_line->file_name, tok_line->line_num, hbaline,
+					  tok_line->err_msg);
 	}
 
 	/* Free tokenizer memory */
@@ -439,7 +444,7 @@ pg_hba_file_rules(PG_FUNCTION_ARGS)
 }
 
 /* Number of columns in pg_ident_file_mappings view */
-#define NUM_PG_IDENT_FILE_MAPPINGS_ATTS	 6
+#define NUM_PG_IDENT_FILE_MAPPINGS_ATTS	 7
 
 /*
  * fill_ident_line: build one row of pg_ident_file_mappings view, add it to
@@ -448,7 +453,8 @@ pg_hba_file_rules(PG_FUNCTION_ARGS)
  * tuple_store: where to store data
  * tupdesc: tuple descriptor for the view
  * map_number: unique identifier among all valid maps
- * lineno: pg_ident.conf line number (must always be valid)
+ * filename: configuration file name (must always be valid)
+ * lineno: line number of configuration file (must always be valid)
  * ident: parsed line data (can be NULL, in which case err_msg should be set)
  * err_msg: error message (NULL if none)
  *
@@ -457,7 +463,7 @@ pg_hba_file_rules(PG_FUNCTION_ARGS)
  */
 static void
 fill_ident_line(Tuplestorestate *tuple_store, TupleDesc tupdesc,
-				int map_number, int lineno, IdentLine *ident,
+				int map_number, char *filename, int lineno, IdentLine *ident,
 				const char *err_msg)
 {
 	Datum		values[NUM_PG_IDENT_FILE_MAPPINGS_ATTS];
@@ -477,6 +483,9 @@ fill_ident_line(Tuplestorestate *tuple_store, TupleDesc tupdesc,
 	else
 		values[index++] = Int32GetDatum(map_number);
 
+	/* file_name */
+	values[index++] = CStringGetTextDatum(filename);
+
 	/* line_number */
 	values[index++] = Int32GetDatum(lineno);
 
@@ -489,7 +498,7 @@ fill_ident_line(Tuplestorestate *tuple_store, TupleDesc tupdesc,
 	else
 	{
 		/* no parsing result, so set relevant fields to nulls */
-		memset(&nulls[2], true, (NUM_PG_IDENT_FILE_MAPPINGS_ATTS - 3) * sizeof(bool));
+		memset(&nulls[3], true, (NUM_PG_IDENT_FILE_MAPPINGS_ATTS - 4) * sizeof(bool));
 	}
 
 	/* error */
@@ -544,8 +553,8 @@ fill_ident_view(Tuplestorestate *tuple_store, TupleDesc tupdesc)
 			map_number++;
 
 		fill_ident_line(tuple_store, tupdesc, map_number,
-						tok_line->line_num, identline,
-						tok_line->err_msg);
+						tok_line->file_name, tok_line->line_num,
+						identline, tok_line->err_msg);
 	}
 
 	/* Free tokenizer memory */
diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index 94da0ee1d74..83f8fe46eba 100644
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -57,6 +57,6 @@
  */
 
 /*							yyyymmddN */
-#define CATALOG_VERSION_NO	202211221
+#define CATALOG_VERSION_NO	202211241
 
 #endif
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index f15aa2dbb1b..f9301b2627e 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -6161,16 +6161,16 @@
 { oid => '3401', descr => 'show pg_hba.conf rules',
   proname => 'pg_hba_file_rules', prorows => '1000', proretset => 't',
   provolatile => 'v', prorettype => 'record', proargtypes => '',
-  proallargtypes => '{int4,int4,text,_text,_text,text,text,text,_text,text}',
-  proargmodes => '{o,o,o,o,o,o,o,o,o,o}',
-  proargnames => '{rule_number,line_number,type,database,user_name,address,netmask,auth_method,options,error}',
+  proallargtypes => '{int4,text,int4,text,_text,_text,text,text,text,_text,text}',
+  proargmodes => '{o,o,o,o,o,o,o,o,o,o,o}',
+  proargnames => '{rule_number,file_name,line_number,type,database,user_name,address,netmask,auth_method,options,error}',
   prosrc => 'pg_hba_file_rules' },
 { oid => '6250', descr => 'show pg_ident.conf mappings',
   proname => 'pg_ident_file_mappings', prorows => '1000', proretset => 't',
   provolatile => 'v', prorettype => 'record', proargtypes => '',
-  proallargtypes => '{int4,int4,text,text,text,text}',
-  proargmodes => '{o,o,o,o,o,o}',
-  proargnames => '{map_number,line_number,map_name,sys_name,pg_username,error}',
+  proallargtypes => '{int4,text,int4,text,text,text,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{map_number,file_name,line_number,map_name,sys_name,pg_username,error}',
   prosrc => 'pg_ident_file_mappings' },
 { oid => '1371', descr => 'view system lock information',
   proname => 'pg_lock_status', prorows => '1000', proretset => 't',
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 7c7adbc0045..37c1c864739 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1339,6 +1339,7 @@ pg_group| SELECT pg_authid.rolname AS groname,
    FROM pg_authid
   WHERE (NOT pg_authid.rolcanlogin);
 pg_hba_file_rules| SELECT a.rule_number,
+    a.file_name,
     a.line_number,
     a.type,
     a.database,
@@ -1348,14 +1349,15 @@ pg_hba_file_rules| SELECT a.rule_number,
     a.auth_method,
     a.options,
     a.error
-   FROM pg_hba_file_rules() a(rule_number, line_number, type, database, user_name, address, netmask, auth_method, options, error);
+   FROM pg_hba_file_rules() a(rule_number, file_name, line_number, type, database, user_name, address, netmask, auth_method, options, error);
 pg_ident_file_mappings| SELECT a.map_number,
+    a.file_name,
     a.line_number,
     a.map_name,
     a.sys_name,
     a.pg_username,
     a.error
-   FROM pg_ident_file_mappings() a(map_number, line_number, map_name, sys_name, pg_username, error);
+   FROM pg_ident_file_mappings() a(map_number, file_name, line_number, map_name, sys_name, pg_username, error);
 pg_indexes| SELECT n.nspname AS schemaname,
     c.relname AS tablename,
     i.relname AS indexname,