From 612f955ac9619b4d10ed2834582628c54f304c74 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov Date: Fri, 29 Oct 2021 07:20:47 +0300 Subject: [PATCH] porting to version 15 and minor improvements to version 14 Fixed minor bugs in patches for version 14. Added patches for version 15. shm_mq_send started accepting one more argument. Solved this problem with #ifdef PG_VERSION_NUM. --- patches/custom_signals_14.0.patch | 11 -- patches/custom_signals_15.0.patch | 206 +++++++++++++++++++++++ patches/runtime_explain_14.0.patch | 10 -- patches/runtime_explain_15.0.patch | 261 +++++++++++++++++++++++++++++ pg_query_state.c | 4 + signal_handler.c | 4 + 6 files changed, 475 insertions(+), 21 deletions(-) create mode 100644 patches/custom_signals_15.0.patch create mode 100644 patches/runtime_explain_15.0.patch diff --git a/patches/custom_signals_14.0.patch b/patches/custom_signals_14.0.patch index fbb83de..9d640cb 100644 --- a/patches/custom_signals_14.0.patch +++ b/patches/custom_signals_14.0.patch @@ -1,14 +1,3 @@ -From f2632ea7cd03119c55b8aa0ef60f529380ca2536 Mon Sep 17 00:00:00 2001 -From: Kovalenko Anastasia -Date: Tue, 24 Aug 2021 16:22:28 +0300 -Subject: [PATCH] custom-signals - ---- - src/backend/storage/ipc/procsignal.c | 94 ++++++++++++++++++++++++++++ - src/backend/tcop/postgres.c | 2 + - src/include/storage/procsignal.h | 17 +++++ - 3 files changed, 113 insertions(+) - diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index defb75a..4245d28 100644 --- a/src/backend/storage/ipc/procsignal.c diff --git a/patches/custom_signals_15.0.patch b/patches/custom_signals_15.0.patch new file mode 100644 index 0000000..9d640cb --- /dev/null +++ b/patches/custom_signals_15.0.patch @@ -0,0 +1,206 @@ +diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c +index defb75a..4245d28 100644 +--- a/src/backend/storage/ipc/procsignal.c ++++ b/src/backend/storage/ipc/procsignal.c +@@ -95,6 +95,13 @@ typedef struct + #define BARRIER_CLEAR_BIT(flags, type) \ + ((flags) &= ~(((uint32) 1) << (uint32) (type))) + ++#define IsCustomProcSignalReason(reason) \ ++ ((reason) >= PROCSIG_CUSTOM_1 && (reason) <= PROCSIG_CUSTOM_N) ++ ++static bool CustomSignalPendings[NUM_CUSTOM_PROCSIGNALS]; ++static bool CustomSignalProcessing[NUM_CUSTOM_PROCSIGNALS]; ++static ProcSignalHandler_type CustomInterruptHandlers[NUM_CUSTOM_PROCSIGNALS]; ++ + static ProcSignalHeader *ProcSignal = NULL; + static ProcSignalSlot *MyProcSignalSlot = NULL; + +@@ -103,6 +110,8 @@ static void CleanupProcSignalState(int status, Datum arg); + static void ResetProcSignalBarrierBits(uint32 flags); + static bool ProcessBarrierPlaceholder(void); + ++static void CheckAndSetCustomSignalInterrupts(void); ++ + /* + * ProcSignalShmemSize + * Compute space needed for procsignal's shared memory +@@ -246,6 +255,36 @@ CleanupProcSignalState(int status, Datum arg) + slot->pss_pid = 0; + } + ++/* ++ * RegisterCustomProcSignalHandler ++ * Assign specific handler of custom process signal with new ++ * ProcSignalReason key. ++ * ++ * This function has to be called in _PG_init function of extensions at the ++ * stage of loading shared preloaded libraries. Otherwise it throws fatal error. ++ * ++ * Return INVALID_PROCSIGNAL if all slots for custom signals are occupied. ++ */ ++ProcSignalReason ++RegisterCustomProcSignalHandler(ProcSignalHandler_type handler) ++{ ++ ProcSignalReason reason; ++ ++ if (!process_shared_preload_libraries_in_progress) ++ ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), ++ errmsg("cannot register custom signal after startup"))); ++ ++ /* Iterate through custom signal slots to find a free one */ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ if (!CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1]) ++ { ++ CustomInterruptHandlers[reason - PROCSIG_CUSTOM_1] = handler; ++ return reason; ++ } ++ ++ return INVALID_PROCSIGNAL; ++} ++ + /* + * SendProcSignal + * Send a signal to a Postgres process +@@ -679,7 +718,62 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) + if (CheckProcSignal(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN)) + RecoveryConflictInterrupt(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN); + ++ CheckAndSetCustomSignalInterrupts(); ++ + SetLatch(MyLatch); + + errno = save_errno; + } ++ ++/* ++ * Handle receipt of an interrupt indicating any of custom process signals. ++ */ ++static void ++CheckAndSetCustomSignalInterrupts() ++{ ++ ProcSignalReason reason; ++ ++ for (reason = PROCSIG_CUSTOM_1; reason <= PROCSIG_CUSTOM_N; reason++) ++ { ++ if (CheckProcSignal(reason)) ++ { ++ ++ /* set interrupt flags */ ++ InterruptPending = true; ++ CustomSignalPendings[reason - PROCSIG_CUSTOM_1] = true; ++ } ++ } ++ ++ SetLatch(MyLatch); ++} ++ ++/* ++ * CheckAndHandleCustomSignals ++ * Check custom signal flags and call handler assigned to that signal ++ * if it is not NULL ++ * ++ * This function is called within CHECK_FOR_INTERRUPTS if interrupt occurred. ++ */ ++void ++CheckAndHandleCustomSignals(void) ++{ ++ int i; ++ ++ /* Check on expiring of custom signals and call its handlers if exist */ ++ for (i = 0; i < NUM_CUSTOM_PROCSIGNALS; i++) ++ { ++ if (!CustomSignalProcessing[i] && CustomSignalPendings[i]) ++ { ++ ProcSignalHandler_type handler; ++ ++ CustomSignalPendings[i] = false; ++ handler = CustomInterruptHandlers[i]; ++ if (handler != NULL) ++ { ++ CustomSignalProcessing[i] = true; ++ handler(); ++ CustomSignalProcessing[i] = false; ++ } ++ } ++ } ++} +\ No newline at end of file +diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c +index 8cea10c..dd77c98 100644 +--- a/src/backend/tcop/postgres.c ++++ b/src/backend/tcop/postgres.c +@@ -3364,6 +3364,8 @@ ProcessInterrupts(void) + if (ParallelMessagePending) + HandleParallelMessages(); + ++ CheckAndHandleCustomSignals(); ++ + if (LogMemoryContextPending) + ProcessLogMemoryContextInterrupt(); + } +diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h +index eec186b..74af186 100644 +--- a/src/include/storage/procsignal.h ++++ b/src/include/storage/procsignal.h +@@ -17,6 +17,8 @@ + #include "storage/backendid.h" + + ++#define NUM_CUSTOM_PROCSIGNALS 64 ++ + /* + * Reasons for signaling a Postgres child process (a backend or an auxiliary + * process, like checkpointer). We can cope with concurrent signals for different +@@ -29,6 +31,8 @@ + */ + typedef enum + { ++ INVALID_PROCSIGNAL = -1, /* Must be first */ ++ + PROCSIG_CATCHUP_INTERRUPT, /* sinval catchup interrupt */ + PROCSIG_NOTIFY_INTERRUPT, /* listen/notify interrupt */ + PROCSIG_PARALLEL_MESSAGE, /* message from cooperating parallel backend */ +@@ -44,6 +48,14 @@ typedef enum + PROCSIG_RECOVERY_CONFLICT_BUFFERPIN, + PROCSIG_RECOVERY_CONFLICT_STARTUP_DEADLOCK, + ++ PROCSIG_CUSTOM_1, ++ /* ++ * PROCSIG_CUSTOM_2, ++ * ..., ++ * PROCSIG_CUSTOM_N-1, ++ */ ++ PROCSIG_CUSTOM_N = PROCSIG_CUSTOM_1 + NUM_CUSTOM_PROCSIGNALS - 1, ++ + NUM_PROCSIGNALS /* Must be last! */ + } ProcSignalReason; + +@@ -56,6 +68,8 @@ typedef enum + */ + PROCSIGNAL_BARRIER_PLACEHOLDER = 0 + } ProcSignalBarrierType; ++/* Handler of custom process signal */ ++typedef void (*ProcSignalHandler_type) (void); + + /* + * prototypes for functions in procsignal.c +@@ -64,12 +78,15 @@ extern Size ProcSignalShmemSize(void); + extern void ProcSignalShmemInit(void); + + extern void ProcSignalInit(int pss_idx); ++extern ProcSignalReason ++ RegisterCustomProcSignalHandler(ProcSignalHandler_type handler); + extern int SendProcSignal(pid_t pid, ProcSignalReason reason, + BackendId backendId); + + extern uint64 EmitProcSignalBarrier(ProcSignalBarrierType type); + extern void WaitForProcSignalBarrier(uint64 generation); + extern void ProcessProcSignalBarrier(void); ++extern void CheckAndHandleCustomSignals(void); + + extern void procsignal_sigusr1_handler(SIGNAL_ARGS); + +-- +2.25.1 + diff --git a/patches/runtime_explain_14.0.patch b/patches/runtime_explain_14.0.patch index 3dbba9a..7904cc2 100644 --- a/patches/runtime_explain_14.0.patch +++ b/patches/runtime_explain_14.0.patch @@ -1,13 +1,3 @@ -From 71a353d4cac663db43c57452f925082a233c0e49 Mon Sep 17 00:00:00 2001 -From: Kovalenko Anastasia -Date: Mon, 23 Aug 2021 15:29:59 +0300 -Subject: [PATCH] runtime-explain - ---- - src/backend/commands/explain.c | 153 ++++++++++++++++++++++++++++----- - src/include/commands/explain.h | 2 + - 2 files changed, 133 insertions(+), 22 deletions(-) - diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 10644dfac4..7106ed4257 100644 --- a/src/backend/commands/explain.c diff --git a/patches/runtime_explain_15.0.patch b/patches/runtime_explain_15.0.patch new file mode 100644 index 0000000..7904cc2 --- /dev/null +++ b/patches/runtime_explain_15.0.patch @@ -0,0 +1,261 @@ +diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c +index 10644dfac4..7106ed4257 100644 +--- a/src/backend/commands/explain.c ++++ b/src/backend/commands/explain.c +@@ -984,14 +984,36 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + char *relname; + char *conname = NULL; + ++ instr_time starttimespan; ++ double total; ++ double ntuples; ++ double ncalls; ++ ++ if (!es->runtime) ++ { + /* Must clean up instrumentation state */ + InstrEndLoop(instr); ++ } ++ ++ /* Collect statistic variables */ ++ if (!INSTR_TIME_IS_ZERO(instr->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, instr->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ ++ total = instr->total + INSTR_TIME_GET_DOUBLE(instr->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan); ++ ntuples = instr->ntuples + instr->tuplecount; ++ ncalls = ntuples + !INSTR_TIME_IS_ZERO(starttimespan); + + /* + * We ignore triggers that were never invoked; they likely aren't + * relevant to the current query type. + */ +- if (instr->ntuples == 0) ++ if (ncalls == 0) + continue; + + ExplainOpenGroup("Trigger", NULL, true, es); +@@ -1017,9 +1039,9 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + appendStringInfo(es->str, " on %s", relname); + if (es->timing) + appendStringInfo(es->str, ": time=%.3f calls=%.0f\n", +- 1000.0 * instr->total, instr->ntuples); ++ 1000.0 * total, ncalls); + else +- appendStringInfo(es->str, ": calls=%.0f\n", instr->ntuples); ++ appendStringInfo(es->str, ": calls=%.0f\n", ncalls); + } + else + { +@@ -1028,9 +1050,8 @@ report_triggers(ResultRelInfo *rInfo, bool show_relname, ExplainState *es) + ExplainPropertyText("Constraint Name", conname, es); + ExplainPropertyText("Relation", relname, es); + if (es->timing) +- ExplainPropertyFloat("Time", "ms", 1000.0 * instr->total, 3, +- es); +- ExplainPropertyFloat("Calls", NULL, instr->ntuples, 0, es); ++ ExplainPropertyFloat("Time", "ms", 1000.0 * total, 3, es); ++ ExplainPropertyFloat("Calls", NULL, ncalls, 0, es); + } + + if (conname) +@@ -1600,8 +1621,11 @@ ExplainNode(PlanState *planstate, List *ancestors, + * instrumentation results the user didn't ask for. But we do the + * InstrEndLoop call anyway, if possible, to reduce the number of cases + * auto_explain has to contend with. ++ * ++ * If flag es->stateinfo is set, i.e. when printing the current execution ++ * state, this step of cleaning up is missed. + */ +- if (planstate->instrument) ++ if (planstate->instrument && !es->runtime) + InstrEndLoop(planstate->instrument); + + if (es->analyze && +@@ -1636,7 +1660,7 @@ ExplainNode(PlanState *planstate, List *ancestors, + ExplainPropertyFloat("Actual Loops", NULL, nloops, 0, es); + } + } +- else if (es->analyze) ++ else if (es->analyze && !es->runtime) + { + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoString(es->str, " (never executed)"); +@@ -1652,6 +1676,75 @@ ExplainNode(PlanState *planstate, List *ancestors, + } + } + ++ /* ++ * Print the progress of node execution at current loop. ++ */ ++ if (planstate->instrument && es->analyze && es->runtime) ++ { ++ instr_time starttimespan; ++ double startup_sec; ++ double total_sec; ++ double rows; ++ double loop_num; ++ bool finished; ++ ++ if (!INSTR_TIME_IS_ZERO(planstate->instrument->starttime)) ++ { ++ INSTR_TIME_SET_CURRENT(starttimespan); ++ INSTR_TIME_SUBTRACT(starttimespan, planstate->instrument->starttime); ++ } ++ else ++ INSTR_TIME_SET_ZERO(starttimespan); ++ startup_sec = 1000.0 * planstate->instrument->firsttuple; ++ total_sec = 1000.0 * (INSTR_TIME_GET_DOUBLE(planstate->instrument->counter) ++ + INSTR_TIME_GET_DOUBLE(starttimespan)); ++ rows = planstate->instrument->tuplecount; ++ loop_num = planstate->instrument->nloops + 1; ++ ++ finished = planstate->instrument->nloops > 0 ++ && !planstate->instrument->running ++ && INSTR_TIME_IS_ZERO(starttimespan); ++ ++ if (!finished) ++ { ++ ExplainOpenGroup("Current loop", "Current loop", true, es); ++ if (es->format == EXPLAIN_FORMAT_TEXT) ++ { ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ appendStringInfo(es->str, ++ " (Current loop: actual time=%.3f..%.3f rows=%.0f, loop number=%.0f)", ++ startup_sec, total_sec, rows, loop_num); ++ else ++ appendStringInfo(es->str, ++ " (Current loop: running time=%.3f actual rows=0, loop number=%.0f)", ++ total_sec, loop_num); ++ } ++ else ++ appendStringInfo(es->str, ++ " (Current loop: actual rows=%.0f, loop number=%.0f)", ++ rows, loop_num); ++ } ++ else ++ { ++ ExplainPropertyFloat("Actual Loop Number", NULL, loop_num, 0, es); ++ if (es->timing) ++ { ++ if (planstate->instrument->running) ++ { ++ ExplainPropertyFloat("Actual Startup Time", NULL, startup_sec, 3, es); ++ ExplainPropertyFloat("Actual Total Time", NULL, total_sec, 3, es); ++ } ++ else ++ ExplainPropertyFloat("Running Time", NULL, total_sec, 3, es); ++ } ++ ExplainPropertyFloat("Actual Rows", NULL, rows, 0, es); ++ } ++ ExplainCloseGroup("Current loop", "Current loop", true, es); ++ } ++ } ++ + /* in text format, first line ends here */ + if (es->format == EXPLAIN_FORMAT_TEXT) + appendStringInfoChar(es->str, '\n'); +@@ -2051,6 +2144,9 @@ ExplainNode(PlanState *planstate, List *ancestors, + + /* Prepare per-worker buffer/WAL usage */ + if (es->workers_state && (es->buffers || es->wal) && es->verbose) ++ /* Show worker detail after query execution */ ++ if (es->analyze && es->verbose && planstate->worker_instrument ++ && !es->runtime) + { + WorkerInstrumentation *w = planstate->worker_instrument; + +@@ -3015,6 +3111,11 @@ show_hash_info(HashState *hashstate, ExplainState *es) + memcpy(&hinstrument, hashstate->hinstrument, + sizeof(HashInstrumentation)); + ++ if (hashstate->hashtable) ++ { ++ ExecHashAccumInstrumentation(&hinstrument, hashstate->hashtable); ++ } ++ + /* + * Merge results from workers. In the parallel-oblivious case, the + * results from all participants should be identical, except where +@@ -3392,20 +3493,16 @@ show_instrumentation_count(const char *qlabel, int which, + if (!es->analyze || !planstate->instrument) + return; + ++ nloops = planstate->instrument->nloops; + if (which == 2) +- nfiltered = planstate->instrument->nfiltered2; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered2 / nloops : 0); + else +- nfiltered = planstate->instrument->nfiltered1; ++ nfiltered = ((nloops > 0) ? planstate->instrument->nfiltered1 / nloops : 0); + nloops = planstate->instrument->nloops; + + /* In text mode, suppress zero counts; they're not interesting enough */ + if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) +- { +- if (nloops > 0) +- ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); +- else +- ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); +- } ++ ExplainPropertyFloat(qlabel, NULL, nfiltered, 0, es); + } + + /* +@@ -3977,15 +4074,27 @@ show_modifytable_info(ModifyTableState *mtstate, List *ancestors, + double insert_path; + double other_path; + +- InstrEndLoop(outerPlanState(mtstate)->instrument); ++ if (!es->runtime) ++ InstrEndLoop(outerPlanState(mtstate)->instrument); + + /* count the number of source rows */ +- total = outerPlanState(mtstate)->instrument->ntuples; +- other_path = mtstate->ps.instrument->ntuples2; +- insert_path = total - other_path; ++ other_path = mtstate->ps.instrument->nfiltered2; ++ ++ /* ++ * Insert occurs after extracting row from subplan and in runtime mode ++ * we can appear between these two operations - situation when ++ * total > insert_path + other_path. Therefore we don't know exactly ++ * whether last row from subplan is inserted. ++ * We don't print inserted tuples in runtime mode in order to not print ++ * inconsistent data ++ */ ++ if (!es->runtime) ++ { ++ total = outerPlanState(mtstate)->instrument->ntuples; ++ insert_path = total - other_path; ++ ExplainPropertyFloat("Tuples Inserted", NULL, insert_path, 0, es); ++ } + +- ExplainPropertyFloat("Tuples Inserted", NULL, +- insert_path, 0, es); + ExplainPropertyFloat("Conflicting Tuples", NULL, + other_path, 0, es); + } +diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h +index e94d9e49cf..6a157b8bc0 100644 +--- a/src/include/commands/explain.h ++++ b/src/include/commands/explain.h +@@ -47,6 +47,8 @@ typedef struct ExplainState + bool summary; /* print total planning and execution timing */ + bool settings; /* print modified settings */ + ExplainFormat format; /* output format */ ++ bool runtime; /* print intermediate state of query execution, ++ not after completion */ + /* state for output formatting --- not reset for each new plan tree */ + int indent; /* current indentation level */ + List *grouping_stack; /* format-specific grouping state */ +-- +2.25.1 + diff --git a/pg_query_state.c b/pg_query_state.c index a4a7d1b..21a406b 100644 --- a/pg_query_state.c +++ b/pg_query_state.c @@ -891,7 +891,11 @@ SendBgWorkerPids(void) msg->pids[i++] = current_pid; } +#if PG_VERSION_NUM <= 140000 shm_mq_send(mqh, msg_len, msg, false); +#else + shm_mq_send(mqh, msg_len, msg, false, true); +#endif UnlockShmem(&tag); } diff --git a/signal_handler.c b/signal_handler.c index 2af69fd..20325e7 100644 --- a/signal_handler.c +++ b/signal_handler.c @@ -168,7 +168,11 @@ shm_mq_send_nonblocking(shm_mq_handle *mqh, Size nbytes, const void *data, Size for(i = 0; i < attempts; i++) { +#if PG_VERSION_NUM <= 140000 res = shm_mq_send(mqh, nbytes, data, true); +#else + res = shm_mq_send(mqh, nbytes, data, true, true); +#endif if(res == SHM_MQ_SUCCESS) break;