8000 Allow foreign and custom joins to handle EvalPlanQual rechecks. · home201448/postgres@325b357 · GitHub
[go: up one dir, main page]
Skip to content

Commit 325b357

Browse files
robertmhaasrobertmhaas
committed
Allow foreign and custom joins to handle EvalPlanQual rechecks.
Commit e7cb7ee provided basic infrastructure for allowing a foreign data wrapper or custom scan provider to replace a join of one or more tables with a scan. However, this infrastructure failed to take into account the need for possible EvalPlanQual rechecks, and ExecScanFetch would fail an assertion (or just overwrite memory) if such a check was attempted for a plan containing a pushed-down join. To fix, adjust the EPQ machinery to skip some processing steps when scanrelid == 0, making those the responsibility of scan's recheck method, which also has the responsibility in this case of correctly populating the relevant slot. To allow foreign scans to gain control in the right place to make use of this new facility, add a new, optional RecheckForeignScan method. Also, allow a foreign scan to have a child plan, which can be used to correctly populate the slot (or perhaps for something else, but this is the only use currently envisioned). KaiGai Kohei, reviewed by Robert Haas, Etsuro Fujita, and Kyotaro Horiguchi.
1 parent 25517ee commit 325b357

File tree

12 files changed

+149
-18
lines changed

12 files changed

+149
-18
lines changed

‎contrib/file_fdw/file_fdw.c

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,8 @@ static ForeignScan *fileGetForeignPlan(PlannerInfo *root,
121121
Oid foreigntableid,
122122
ForeignPath *best_path,
123123
List *tlist,
124-
List *scan_clauses);
124+
List *scan_clauses,
125+
Plan *outer_plan);
125126
static void fileExplainForeignScan(ForeignScanState *node, ExplainState *es);
126127
static void fileBeginForeignScan(ForeignScanState *node, int eflags);
127128
static TupleTableSlot *fileIterateForeignScan(ForeignScanState *node);
@@ -525,6 +526,7 @@ fileGetForeignPaths(PlannerInfo *root,
525526
total_cost,
526527
NIL, /* no pathkeys */
527528
NULL, /* no outer rel either */
529+
NULL, /* no extra plan */
528530
coptions));
529531

530532
/*
@@ -544,7 +546,8 @@ fileGetForeignPlan(PlannerInfo *root,
544546
Oid foreigntableid,
545547
ForeignPath *best_path,
546548
List *tlist,
547-
List *scan_clauses)
549+
List *scan_clauses,
550+
Plan *outer_plan)
548551
{
549552
Index scan_relid = baserel->relid;
550553

@@ -564,7 +567,8 @@ fileGetForeignPlan(PlannerInfo *root,
564567
NIL, /* no expressions to evaluate */
565568
best_path->fdw_private,
566569
NIL, /* no custom tlist */
567-
NIL /* no remote quals */ );
570+
NIL, /* no remote quals */
571+
outer_plan);
568572
}
569573

570574
/*

‎contrib/postgres_fdw/postgres_fdw.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,8 @@ static ForeignScan *postgresGetForeignPlan(PlannerInfo *root,
245245
Oid foreigntableid,
246246
ForeignPath *best_path,
247247
List *tlist,
248-
List *scan_clauses);
248+
List *scan_clauses,
249+
Plan *outer_plan);
249250
static void postgresBeginForeignScan(ForeignScanState *node, int eflags);
250251
static TupleTableSlot *postgresIterateForeignScan(ForeignScanState *node);
251252
static void postgresReScanForeignScan(ForeignScanState *node);
@@ -560,6 +561,7 @@ postgresGetForeignPaths(PlannerInfo *root,
560561
fpinfo->total_cost,
561562
NIL, /* no pathkeys */
562563
NULL, /* no outer rel either */
564+
NULL, /* no extra plan */
563565
NIL); /* no fdw_private list */
564566
add_path(baserel, (Path *) path);
565567

@@ -727,6 +729,7 @@ postgresGetForeignPaths(PlannerInfo *root,
727729
total_cost,
728730
NIL, /* no pathkeys */
729731
param_info->ppi_req_outer,
732+
NULL,
730733
NIL); /* no fdw_private list */
731734
add_path(baserel, (Path *) path);
732735
}
@@ -742,7 +745,8 @@ postgresGetForeignPlan(PlannerInfo *root,
742745
Oid foreigntableid,
743746
ForeignPath *best_path,
744747
List *tlist,
745-
List *scan_clauses)
748+
List *scan_clauses,
749+
Plan *outer_plan)
746750
{
747751
PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) baserel->fdw_private;
748752
Index scan_relid = baserel->relid;
@@ -882,7 +886,8 @@ postgresGetForeignPlan(PlannerInfo *root,
882886
params_list,
883887
fdw_private,
884888
NIL, /* no custom tlist */
885-
remote_exprs);
889+
remote_exprs,
890+
outer_plan);
886891
}
887892

888893
/*

‎doc/src/sgml/fdwhandler.sgml

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,8 @@ GetForeignPlan (PlannerInfo *root,
168168
Oid foreigntableid,
169169
ForeignPath *best_path,
170170
List *tlist,
171-
List *scan_clauses);
171+
List *scan_clauses,
172+
Plan *outer_plan);
172173
</programlisting>
173174

174175
Create a <structname>ForeignScan</> plan node from the selected foreign
@@ -765,6 +766,35 @@ RefetchForeignRow (EState *estate,
765766
See <xref linkend="fdw-row-locking"> for more information.
766767
</para>
767768

769+
<para>
770+
<programlisting>
771+
bool
772+
RecheckForeignScan (ForeignScanState *node, TupleTableSlot *slot);
773+
</programlisting>
774+
Recheck that a previously-returned tuple still matches the relevant
775+
scan and join qualifiers, and possibly provide a modified version of
776+
the tuple. For foreign data wrappers which do not perform join pushdown,
777+
it will typically be more convenient to set this to <literal>NULL</> and
778+
instead set <structfield>fdw_recheck_quals</structfield> appropriately.
779+
When outer joins are pushed down, however, it isn't sufficient to
780+
reapply the checks relevant to all the base tables to the result tuple,
781+
even if all needed attributes are present, because failure to match some
782+
qualifier might result in some attributes going to NULL, rather than in
783+
no tuple being returned. <literal>RecheckForeignScan</> can recheck
784+
qualifiers and return true if they are still satisfied and false
785+
otherwise, but it can also store a replacement tuple into the supplied
786+
slot.
787+
</para>
788+
789+
<para>
790+
To implement join pushdown, a foreign data wrapper will typically
791+
construct an alternative local join plan which is used only for
792+
rechecks; this will become the outer subplan of the
793+
<literal>ForeignScan</>. When a recheck is required, this subplan
794+
can be executed and the resulting tuple can be stored in the slot.
795+
This plan need not be efficient since no base table will return more
796+
that one row; for example, it may implement all joins as nested loops.
797+
</para>
768798
</sect2>
769799

770800
<sect2 id="fdw-callbacks-explain">
@@ -1137,11 +1167,17 @@ GetForeignServerByName(const char *name, bool missing_ok);
11371167

11381168
<para>
11391169
Any clauses removed from the plan node's qual list must instead be added
1140-
to <literal>fdw_recheck_quals</> in order to ensure correct behavior
1170+
to <literal>fdw_recheck_quals</> or rechecked by
1171+
<literal>RecheckForeignScan</> in order to ensure correct behavior
11411172
at the <literal>READ COMMITTED</> isolation level. When a concurrent
11421173
update occurs for some other table involved in the query, the executor
11431174
may need to verify that all of the original quals are still satisfied for
1144-
the tuple, possibly against a different set of parameter values.
1175+
the tuple, possibly against a different set of parameter values. Using
1176+
<literal>fdw_recheck_quals</> is typically easier than implementing checks
1177+
inside <literal>RecheckForeignScan</>, but this method will be
1178+
insufficient when outer joins have been pushed down, since the join tuples
1179+
in that case might have some fields go to NULL without rejecting the
1180+
tuple entirely.
11451181
</para>
11461182

11471183
<para>

‎src/backend/executor/execScan.c

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,21 @@ ExecScanFetch(ScanState *node,
4949
*/
5050
Index scanrelid = ((Scan *) node->ps.plan)->scanrelid;
5151

52-
Assert(scanrelid > 0);
53-
if (estate->es_epqTupleSet[scanrelid - 1])
52+
if (scanrelid == 0)
53+
{
54+
TupleTableSlot *slot = node->ss_ScanTupleSlot;
55+
56+
/*
57+
* This is a ForeignScan or CustomScan which has pushed down a
58+
* join to the remote side. The recheck method is responsible not
59+
* only for rechecking the scan/join quals but also for storing
60+
* the correct tuple in the slot.
61+
*/
62+
if (!(*recheckMtd) (node, slot))
63+
ExecClearTuple(slot); /* would not be returned by scan */
64+
return slot;
65+
}
66+
else if (estate->es_epqTupleSet[scanrelid - 1])
5467
{
5568
TupleTableSlot *slot = node->ss_ScanTupleSlot;
5669

@@ -347,8 +360,31 @@ ExecScanReScan(ScanState *node)
347360
{
348361
Index scanrelid = ((Scan *) node->ps.plan)->scanrelid;
349362

350-
Assert(scanrelid > 0);
363+
if (scanrelid > 0)
364+
estate->es_epqScanDone[scanrelid - 1] = false;
365+
else
366+
{
367+
Bitmapset *relids;
368+
int rtindex = -1;
351369

352-
estate->es_epqScanDone[scanrelid - 1] = false;
370+
/*
371+
* If an FDW or custom scan provider has replaced the join with a
372+
* scan, there are multiple RTIs; reset the epqScanDone flag for
373+
* all of them.
374+
*/
375+
if (IsA(node->ps.plan, ForeignScan))
376+
relids = ((ForeignScan *) node->ps.plan)->fs_relids;
377+
else if (IsA(node->ps.plan, CustomScan))
378+
relids = ((CustomScan *) node->ps.plan)->custom_relids;
379+
else
380+
elog(ERROR, "unexpected scan node: %d",
381+
(int) nodeTag(node->ps.plan));
382+
383+
while ((rtindex = bms_next_member(relids, rtindex)) >= 0)
384+
{
385+
Assert(rtindex > 0);
386+
estate->es_epqScanDone[rtindex - 1] = false;
387+
}
388+
}
353389
}
354390
}

‎src/backend/executor/nodeForeignscan.c

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ ForeignNext(ForeignScanState *node)
7373
static bool
7474
ForeignRecheck(ForeignScanState *node, TupleTableSlot *slot)
7575
{
76+
FdwRoutine *fdwroutine = node->fdwroutine;
7677
ExprContext *econtext;
7778

7879
/*
@@ -85,6 +86,18 @@ ForeignRecheck(ForeignScanState *node, TupleTableSlot *slot)
8586

8687
ResetExprContext(econtext);
8788

89+
/*
90+
* If an outer join is pushed down, RecheckForeignScan may need to store a
91+
* different tuple in the slot, because a different set of columns may go
92+
* to NULL upon recheck. Otherwise, it shouldn't need to change the slot
93+
* contents, just return true or false to indicate whether the quals still
94+
* pass. For simple cases, setting fdw_recheck_quals may be easier than
95+
* providing this callback.
96+
*/
97+
if (fdwroutine->RecheckForeignScan &&
98+
!fdwroutine->RecheckForeignScan(node, slot))
99+
return false;
100+
88101
return ExecQual(node->fdw_recheck_quals, econtext, false);
89102
}
90103

@@ -205,6 +218,11 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
205218
scanstate->fdwroutine = fdwroutine;
206219
scanstate->fdw_state = NULL;
207220

221+
/* Initialize any outer plan. */
222+
if (outerPlan(node))
223+
outerPlanState(scanstate) =
224+
ExecInitNode(outerPlan(node), estate, eflags);
225+
208226
/*
209227
* Tell the FDW to initialize the scan.
210228
*/
@@ -225,6 +243,10 @@ ExecEndForeignScan(ForeignScanState *node)
225243
/* Let the FDW shut down */
226244
node->fdwroutine->EndForeignScan(node);
227245

246+
/* Shut down any outer plan. */
247+
if (outerPlanState(node))
248+
ExecEndNode(outerPlanState(node));
249+
228250
/* Free the exprcontext */
229251
ExecFreeExprContext(&node->ss.ps);
230252

@@ -246,7 +268,17 @@ ExecEndForeignScan(ForeignScanState *node)
246268
void
247269
ExecReScanForeignScan(ForeignScanState *node)
248270
{
271+
PlanState *outerPlan = outerPlanState(node);
272+
249273
node->fdwroutine->ReScanForeignScan(node);
250274

275+
/*
276+
* If chgParam of subnode is not null then plan will be re-scanned by
277+
* first ExecProcNode. outerPlan may also be NULL, in which case there
278+
* is nothing to rescan at all.
279+
*/
280+
if (outerPlan != NULL && outerPlan->chgParam == NULL)
281+
ExecReScan(outerPlan);
282+
251283
ExecScanReScan(&node->ss);
252284
}

‎src/backend/nodes/outfuncs.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1665,6 +1665,7 @@ _outForeignPath(StringInfo str, const ForeignPath *node)
16651665

16661666
_outPathInfo(str, (const Path *) node);
16671667

1668+
WRITE_NODE_FIELD(fdw_outerpath);
16681669
WRITE_NODE_FIELD(fdw_private);
16691670
}
16701671

‎src/backend/optimizer/plan/createplan.c

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2059,11 +2059,16 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
20592059
Index scan_relid = rel->relid;
20602060
Oid rel_oid = InvalidOid;
20612061
Bitmapset *attrs_used = NULL;
2062+
Plan *outer_plan = NULL;
20622063
ListCell *lc;
20632064
int i;
20642065

20652066
Assert(rel->fdwroutine != NULL);
20662067

2068+
/* transform the child path if any */
2069+
if (best_path->fdw_outerpath)
2070+
outer_plan = create_plan_recurse(root, best_path->fdw_outerpath);
2071+
20672072
/*
20682073
* If we're scanning a base relation, fetch its OID. (Irrelevant if
20692074
* scanning a join relation.)
@@ -2093,7 +2098,8 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
20932098
*/
20942099
scan_plan = rel->fdwroutine->GetForeignPlan(root, rel, rel_oid,
20952100
best_path,
2096-
tlist, scan_clauses);
2101+
tlist, scan_clauses,
2102+
outer_plan);
20972103

20982104
/* Copy cost data from Path to Plan; no need to make FDW do this */
20992105
copy_path_costsize(&scan_plan->scan.plan, &best_path->path);
@@ -3706,15 +3712,16 @@ make_foreignscan(List *qptlist,
37063712
List *fdw_exprs,
37073713
List *fdw_private,
37083714
List *fdw_scan_tlist,
3709-
List *fdw_recheck_quals)
3715+
List *fdw_recheck_quals,
3716+
Plan *outer_plan)
37103717
{
37113718
ForeignScan *node = makeNode(ForeignScan);
37123719
Plan *plan = &node->scan.plan;
37133720

37143721
/* cost will be filled in by create_foreignscan_plan */
37153722
plan->targetlist = qptlist;
37163723
plan->qual = qpqual;
3717-
plan->lefttree = NULL;
3724+
plan->lefttree = outer_plan;
37183725
plan->righttree = NULL;
37193726
node->scan.scanrelid = scanrelid;
37203727
/* fs_server will be filled in by create_foreignscan_plan */

‎src/backend/optimizer/util/pathnode.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1462,6 +1462,7 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel,
14621462
double rows, Cost startup_cost, Cost total_cost,
14631463
List *pathkeys,
14641464
Relids required_outer,
1465+
Path *fdw_outerpath,
14651466
List *fdw_private)
14661467
{
14671468
ForeignPath *pathnode = makeNode(ForeignPath);
@@ -1475,6 +1476,7 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel,
14751476
pathnode->path.total_cost = total_cost;
14761477
pathnode->path.pathkeys = pathkeys;
14771478

1479+
pathnode->fdw_outerpath = fdw_outerpath;
14781480
pathnode->fdw_private = fdw_private;
14791481

14801482
return pathnode;

‎src/include/foreign/fdwapi.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,17 @@ typedef ForeignScan *(*GetForeignPlan_function) (PlannerInfo *root,
3636
Oid foreigntableid,
3737
ForeignPath *best_path,
3838
List *tlist,
39-
List *scan_clauses);
39+
List *scan_clauses,
40+
Plan *outer_plan);
4041

4142
typedef void (*BeginForeignScan_function) (ForeignScanState *node,
4243
int eflags);
4344

4445
typedef TupleTableSlot *(*IterateForeignScan_function) (ForeignScanState *node);
4546

47+
typedef bool (*RecheckForeignScan_function) (ForeignScanState *node,
48+
TupleTableSlot *slot);
49+
4650
typedef void (*ReScanForeignScan_function) (ForeignScanState *node);
4751

4852
typedef void (*EndForeignScan_function) (ForeignScanState *node);
@@ -162,6 +166,7 @@ typedef struct FdwRoutine
162166
/* Functions for SELECT FOR UPDATE/SHARE row locking */
163167
GetForeignRowMarkType_function GetForeignRowMarkType;
164168
RefetchForeignRow_function RefetchForeignRow;
169+
RecheckForeignScan_function RecheckForeignScan;
165170

166171
/* Support functions for EXPLAIN */
167172
ExplainForeignScan_function ExplainForeignScan;

‎src/include/nodes/relation.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -894,6 +894,7 @@ typedef struct TidPath
894894
typedef struct ForeignPath
895895
{
896896
Path path;
897+
Path *fdw_outerpath;
897898
List *fdw_private;
898899
} ForeignPath;
899900

‎src/include/optimizer/pathnode.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ extern ForeignPath *create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel,
8383
double rows, Cost startup_cost, Cost total_cost,
8484
List *pathkeys,
8585
Relids required_outer,
86+
Path *fdw_outerpath,
8687
List *fdw_private);
8788

8889
extern Relids calc_nestloop_required_outer(Path *outer_path, Path *inner_path);

‎src/include/optimizer/planmain.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ extern SubqueryScan *make_subqueryscan(List *qptlist, List *qpqual,
4545
Index scanrelid, Plan *subplan);
4646
extern ForeignScan *make_foreignscan(List *qptlist, List *qpqual,
4747
Index scanrelid, List *fdw_exprs, List *fdw_private,
48-
List *fdw_scan_tlist, List *fdw_recheck_quals);
48+
List *fdw_scan_tlist, List *fdw_recheck_quals,
49+
Plan *outer_plan);
4950
extern Append *make_append(List *appendplans, List *tlist);
5051
extern RecursiveUnion *make_recursive_union(List *tlist,
5152
Plan *lefttree, Plan *righttree, int wtParam,

0 commit comments

Comments
 (0)
0