Allow foreign and custom joins to handle EvalPlanQual rechecks.

Commit e7cb7ee14555cc9c5773e2c102efd6371f6f2005 provided basic
infrastructure for allowing a foreign data wrapper or custom scan
provider to replace a join of one or more tables with a scan.
However, this infrastructure failed to take into account the need
for possible EvalPlanQual rechecks, and ExecScanFetch would fail
an assertion (or just overwrite memory) if such a check was attempted
for a plan containing a pushed-down join.  To fix, adjust the EPQ
machinery to skip some processing steps when scanrelid == 0, making
those the responsibility of scan's recheck method, which also has
the responsibility in this case of correctly populating the relevant
slot.

To allow foreign scans to gain control in the right place to make
use of this new facility, add a new, optional RecheckForeignScan
method.  Also, allow a foreign scan to have a child plan, which can
be used to correctly populate the slot (or perhaps for something
else, but this is the only use currently envisioned).

KaiGai Kohei, reviewed by Robert Haas, Etsuro Fujita, and Kyotaro
Horiguchi.

9 files changed, 95 insertions, 9 deletions

diff --git a/src/backend/executor/execScan.c b/src/backend/executor/execScan.c
index a96e826ba42..3faf7f9a77d 100644
--- a/src/backend/executor/execScan.c
+++ b/src/backend/executor/execScan.c
@@ -49,8 +49,21 @@ ExecScanFetch(ScanState *node,
 		 */
 		Index		scanrelid = ((Scan *) node->ps.plan)->scanrelid;
 
-		Assert(scanrelid > 0);
-		if (estate->es_epqTupleSet[scanrelid - 1])
+		if (scanrelid == 0)
+		{
+			TupleTableSlot *slot = node->ss_ScanTupleSlot;
+
+			/*
+			 * This is a ForeignScan or CustomScan which has pushed down a
+			 * join to the remote side.  The recheck method is responsible not
+			 * only for rechecking the scan/join quals but also for storing
+			 * the correct tuple in the slot.
+			 */
+			if (!(*recheckMtd) (node, slot))
+				ExecClearTuple(slot);	/* would not be returned by scan */
+			return slot;
+		}
+		else if (estate->es_epqTupleSet[scanrelid - 1])
 		{
 			TupleTableSlot *slot = node->ss_ScanTupleSlot;
 
@@ -347,8 +360,31 @@ ExecScanReScan(ScanState *node)
 	{
 		Index		scanrelid = ((Scan *) node->ps.plan)->scanrelid;
 
-		Assert(scanrelid > 0);
+		if (scanrelid > 0)
+			estate->es_epqScanDone[scanrelid - 1] = false;
+		else
+		{
+			Bitmapset  *relids;
+			int			rtindex = -1;
 
-		estate->es_epqScanDone[scanrelid - 1] = false;
+			/*
+			 * If an FDW or custom scan provider has replaced the join with a
+			 * scan, there are multiple RTIs; reset the epqScanDone flag for
+			 * all of them.
+			 */
+			if (IsA(node->ps.plan, ForeignScan))
+				relids = ((ForeignScan *) node->ps.plan)->fs_relids;
+			else if (IsA(node->ps.plan, CustomScan))
+				relids = ((CustomScan *) node->ps.plan)->custom_relids;
+			else
+				elog(ERROR, "unexpected scan node: %d",
+					 (int) nodeTag(node->ps.plan));
+
+			while ((rtindex = bms_next_member(relids, rtindex)) >= 0)
+			{
+				Assert(rtindex > 0);
+				estate->es_epqScanDone[rtindex - 1] = false;
+			}
+		}
 	}
 }
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 6165e4a6cb4..62959e3a649 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -73,6 +73,7 @@ ForeignNext(ForeignScanState *node)
 static bool
 ForeignRecheck(ForeignScanState *node, TupleTableSlot *slot)
 {
+	FdwRoutine *fdwroutine = node->fdwroutine;
 	ExprContext *econtext;
 
 	/*
@@ -85,6 +86,18 @@ ForeignRecheck(ForeignScanState *node, TupleTableSlot *slot)
 
 	ResetExprContext(econtext);
 
+	/*
+	 * If an outer join is pushed down, RecheckForeignScan may need to store a
+	 * different tuple in the slot, because a different set of columns may go
+	 * to NULL upon recheck.  Otherwise, it shouldn't need to change the slot
+	 * contents, just return true or false to indicate whether the quals still
+	 * pass.  For simple cases, setting fdw_recheck_quals may be easier than
+	 * providing this callback.
+	 */
+	if (fdwroutine->RecheckForeignScan &&
+		!fdwroutine->RecheckForeignScan(node, slot))
+		return false;
+
 	return ExecQual(node->fdw_recheck_quals, econtext, false);
 }
 
@@ -205,6 +218,11 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	scanstate->fdwroutine = fdwroutine;
 	scanstate->fdw_state = NULL;
 
+	/* Initialize any outer plan. */
+	if (outerPlan(node))
+		outerPlanState(scanstate) =
+			ExecInitNode(outerPlan(node), estate, eflags);
+
 	/*
 	 * Tell the FDW to initialize the scan.
 	 */
@@ -225,6 +243,10 @@ ExecEndForeignScan(ForeignScanState *node)
 	/* Let the FDW shut down */
 	node->fdwroutine->EndForeignScan(node);
 
+	/* Shut down any outer plan. */
+	if (outerPlanState(node))
+		ExecEndNode(outerPlanState(node));
+
 	/* Free the exprcontext */
 	ExecFreeExprContext(&node->ss.ps);
 
@@ -246,7 +268,17 @@ ExecEndForeignScan(ForeignScanState *node)
 void
 ExecReScanForeignScan(ForeignScanState *node)
 {
+	PlanState  *outerPlan = outerPlanState(node);
+
 	node->fdwroutine->ReScanForeignScan(node);
 
+	/*
+	 * If chgParam of subnode is not null then plan will be re-scanned by
+	 * first ExecProcNode.  outerPlan may also be NULL, in which case there
+	 * is nothing to rescan at all.
+	 */
+	if (outerPlan != NULL && outerPlan->chgParam == NULL)
+		ExecReScan(outerPlan);
+
 	ExecScanReScan(&node->ss);
 }
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index c709105d4fe..f07c7933b14 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -1683,6 +1683,7 @@ _outForeignPath(StringInfo str, const ForeignPath *node)
 
 	_outPathInfo(str, (const Path *) node);
 
+	WRITE_NODE_FIELD(fdw_outerpath);
 	WRITE_NODE_FIELD(fdw_private);
 }
 
diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index 411b36c418e..32f903d8dca 100644
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -2095,11 +2095,16 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
 	Index		scan_relid = rel->relid;
 	Oid			rel_oid = InvalidOid;
 	Bitmapset  *attrs_used = NULL;
+	Plan	   *outer_plan = NULL;
 	ListCell   *lc;
 	int			i;
 
 	Assert(rel->fdwroutine != NULL);
 
+	/* transform the child path if any */
+	if (best_path->fdw_outerpath)
+		outer_plan = create_plan_recurse(root, best_path->fdw_outerpath);
+
 	/*
 	 * If we're scanning a base relation, fetch its OID.  (Irrelevant if
 	 * scanning a join relation.)
@@ -2129,7 +2134,8 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
 	 */
 	scan_plan = rel->fdwroutine->GetForeignPlan(root, rel, rel_oid,
 												best_path,
-												tlist, scan_clauses);
+												tlist, scan_clauses,
+												outer_plan);
 
 	/* Copy cost data from Path to Plan; no need to make FDW do this */
 	copy_generic_path_info(&scan_plan->scan.plan, &best_path->path);
@@ -3747,7 +3753,8 @@ make_foreignscan(List *qptlist,
 				 List *fdw_exprs,
 				 List *fdw_private,
 				 List *fdw_scan_tlist,
-				 List *fdw_recheck_quals)
+				 List *fdw_recheck_quals,
+				 Plan *outer_plan)
 {
 	ForeignScan *node = makeNode(ForeignScan);
 	Plan	   *plan = &node->scan.plan;
@@ -3755,7 +3762,7 @@ make_foreignscan(List *qptlist,
 	/* cost will be filled in by create_foreignscan_plan */
 	plan->targetlist = qptlist;
 	plan->qual = qpqual;
-	plan->lefttree = NULL;
+	plan->lefttree = outer_plan;
 	plan->righttree = NULL;
 	node->scan.scanrelid = scanrelid;
 	/* fs_server will be filled in by create_foreignscan_plan */
diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c
index 09c32445462..ec0910dbb4f 100644
--- a/src/backend/optimizer/util/pathnode.c
+++ b/src/backend/optimizer/util/pathnode.c
@@ -1507,6 +1507,7 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel,
 						double rows, Cost startup_cost, Cost total_cost,
 						List *pathkeys,
 						Relids required_outer,
+						Path *fdw_outerpath,
 						List *fdw_private)
 {
 	ForeignPath *pathnode = makeNode(ForeignPath);
@@ -1521,6 +1522,7 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel,
 	pathnode->path.total_cost = total_cost;
 	pathnode->path.pathkeys = pathkeys;
 
+	pathnode->fdw_outerpath = fdw_outerpath;
 	pathnode->fdw_private = fdw_private;
 
 	return pathnode;
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 69b48b46778..e9fdacd4498 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -36,13 +36,17 @@ typedef ForeignScan *(*GetForeignPlan_function) (PlannerInfo *root,
 														  Oid foreigntableid,
 													  ForeignPath *best_path,
 															 List *tlist,
-														 List *scan_clauses);
+														  List *scan_clauses,
+														   Plan *outer_plan);
 
 typedef void (*BeginForeignScan_function) (ForeignScanState *node,
 													   int eflags);
 
 typedef TupleTableSlot *(*IterateForeignScan_function) (ForeignScanState *node);
 
+typedef bool (*RecheckForeignScan_function) (ForeignScanState *node,
+													   TupleTableSlot *slot);
+
 typedef void (*ReScanForeignScan_function) (ForeignScanState *node);
 
 typedef void (*EndForeignScan_function) (ForeignScanState *node);
@@ -162,6 +166,7 @@ typedef struct FdwRoutine
 	/* Functions for SELECT FOR UPDATE/SHARE row locking */
 	GetForeignRowMarkType_function GetForeignRowMarkType;
 	RefetchForeignRow_function RefetchForeignRow;
+	RecheckForeignScan_function RecheckForeignScan;
 
 	/* Support functions for EXPLAIN */
 	ExplainForeignScan_function ExplainForeignScan;
diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h
index 6de07a1fbd0..32321238ae4 100644
--- a/src/include/nodes/relation.h
+++ b/src/include/nodes/relation.h
@@ -911,6 +911,7 @@ typedef struct TidPath
 typedef struct ForeignPath
 {
 	Path		path;
+	Path	   *fdw_outerpath;
 	List	   *fdw_private;
 } ForeignPath;
 
diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h
index eea75d0d1ec..f41847f4093 100644
--- a/src/include/optimizer/pathnode.h
+++ b/src/include/optimizer/pathnode.h
@@ -86,6 +86,7 @@ extern ForeignPath *create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel,
 						double rows, Cost startup_cost, Cost total_cost,
 						List *pathkeys,
 						Relids required_outer,
+						Path *fdw_outerpath,
 						List *fdw_private);
 
 extern Relids calc_nestloop_required_outer(Path *outer_path, Path *inner_path);
diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h
index 1fb850489fb..f96e9ee5054 100644
--- a/src/include/optimizer/planmain.h
+++ b/src/include/optimizer/planmain.h
@@ -45,7 +45,8 @@ extern SubqueryScan *make_subqueryscan(List *qptlist, List *qpqual,
 				  Index scanrelid, Plan *subplan);
 extern ForeignScan *make_foreignscan(List *qptlist, List *qpqual,
 				 Index scanrelid, List *fdw_exprs, List *fdw_private,
-				 List *fdw_scan_tlist, List *fdw_recheck_quals);
+				 List *fdw_scan_tlist, List *fdw_recheck_quals,
+				 Plan *outer_plan);
 extern Append *make_append(List *appendplans, List *tlist);
 extern RecursiveUnion *make_recursive_union(List *tlist,
 					 Plan *lefttree, Plan *righttree, int wtParam,