From 2c843c8220ede1ed96593440232d5430a72f461a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Thu, 21 Jan 2016 11:11:19 +0100 Subject: [PATCH 01/31] Release 9.5.0-2cdb1 --- debian/changelog | 8 + debian/patches/90-profile_fdw.patch | 60 ++ debian/patches/91-extension_shipping.patch | 796 +++++++++++++++++++++ debian/patches/series | 2 + 4 files changed, 866 insertions(+) create mode 100644 debian/patches/90-profile_fdw.patch create mode 100644 debian/patches/91-extension_shipping.patch diff --git a/debian/changelog b/debian/changelog index 5ef21ff..b36229c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +postgresql-9.5 (9.5.0-2cdb1) precise; urgency=low + + * Backport FDW extension shipping + * Add FDW timing debug profiling + * Release for CartoDB + + -- keybase.io/zenitram Wed, 20 Jan 2016 11:23:06 +0000 + postgresql-9.5 (9.5.0-2) unstable; urgency=medium * Disable spinlocks on alpha. Performance will be poor, but at least they diff --git a/debian/patches/90-profile_fdw.patch b/debian/patches/90-profile_fdw.patch new file mode 100644 index 0000000..7fd73af --- /dev/null +++ b/debian/patches/90-profile_fdw.patch @@ -0,0 +1,60 @@ +From 021a3748ddbffdb42afca05c9ebde8c9862e4fba Mon Sep 17 00:00:00 2001 +From: Paul Ramsey +Date: Fri, 8 Jan 2016 11:35:45 -0800 +Subject: [PATCH 1/2] Provide FDW query timing on DEBUG + +--- + contrib/postgres_fdw/postgres_fdw.c | 19 ++++++++++++++++++- + 1 file changed, 18 insertions(+), 1 deletion(-) + +Index: postgresql-9.5.0/contrib/postgres_fdw/postgres_fdw.c +=================================================================== +--- postgresql-9.5.0.orig/contrib/postgres_fdw/postgres_fdw.c 2016-01-20 14:40:31.683535083 +0000 ++++ postgresql-9.5.0/contrib/postgres_fdw/postgres_fdw.c 2016-01-20 14:50:49.418538947 +0000 +@@ -39,6 +39,9 @@ + #include "utils/rel.h" + #include "utils/sampling.h" + ++#include ++#include ++ + PG_MODULE_MAGIC; + + /* Default CPU cost to start up a foreign query. */ +@@ -154,6 +157,9 @@ + int fetch_ct_2; /* Min(# of fetches done, 2) */ + bool eof_reached; /* true if last fetch reached EOF */ + ++ /* Timing */ ++ struct timeval start_time; ++ + /* working memory contexts */ + MemoryContext batch_cxt; /* context holding current batch of tuples */ + MemoryContext temp_cxt; /* context for per-tuple temporary data */ +@@ -1019,7 +1025,10 @@ + * cursor on the remote side. + */ + if (!fsstate->cursor_exists) ++ { ++ gettimeofday(&(fsstate->start_time), NULL); + create_cursor(node); ++ } + + /* + * Get some more tuples, if we've run out. +@@ -1116,7 +1125,15 @@ + + /* Close the cursor if open, to prevent accumulation of cursors */ + if (fsstate->cursor_exists) ++ { ++ struct timeval end_time; ++ double secs; ++ gettimeofday(&end_time, NULL); ++ secs = 1000 * end_time.tv_sec + end_time.tv_usec/1000.0; ++ secs -= 1000 * (fsstate->start_time).tv_sec + (fsstate->start_time).tv_usec/1000.0; ++ elog(DEBUG1, "FDW Query Duration: %.3lf ms", secs); + close_cursor(fsstate->conn, fsstate->cursor_number); ++ } + + /* Release remote connection */ + ReleaseConnection(fsstate->conn); diff --git a/debian/patches/91-extension_shipping.patch b/debian/patches/91-extension_shipping.patch new file mode 100644 index 0000000..c96d01c --- /dev/null +++ b/debian/patches/91-extension_shipping.patch @@ -0,0 +1,796 @@ +From 2c11c48f0b057b4702c2d1be8e21058f81134bb3 Mon Sep 17 00:00:00 2001 +From: Paul Ramsey +Date: Thu, 7 Jan 2016 11:27:20 -0800 +Subject: [PATCH] Allow postgres_fdw to ship extension funcs/operators for + remote execution + +--- + contrib/postgres_fdw/Makefile | 2 +- + contrib/postgres_fdw/deparse.c | 92 ++++++++-------- + contrib/postgres_fdw/option.c | 54 +++++++++ + contrib/postgres_fdw/postgres_fdw.c | 38 +------ + contrib/postgres_fdw/postgres_fdw.h | 41 +++++++ + contrib/postgres_fdw/shippable.c | 214 ++++++++++++++++++++++++++++++++++++ + doc/src/sgml/postgres-fdw.sgml | 46 +++++++- + src/backend/utils/adt/format_type.c | 16 ++- + src/include/utils/builtins.h | 1 + + 9 files changed, 420 insertions(+), 84 deletions(-) + create mode 100644 contrib/postgres_fdw/shippable.c + +Index: postgresql-9.5.0/contrib/postgres_fdw/Makefile +=================================================================== +--- postgresql-9.5.0.orig/contrib/postgres_fdw/Makefile 2016-01-20 14:40:31.000000000 +0000 ++++ postgresql-9.5.0/contrib/postgres_fdw/Makefile 2016-01-20 14:50:42.518332653 +0000 +@@ -1,7 +1,7 @@ + # contrib/postgres_fdw/Makefile + + MODULE_big = postgres_fdw +-OBJS = postgres_fdw.o option.o deparse.o connection.o $(WIN32RES) ++OBJS = postgres_fdw.o option.o deparse.o connection.o shippable.o $(WIN32RES) + PGFILEDESC = "postgres_fdw - foreign data wrapper for PostgreSQL" + + PG_CPPFLAGS = -I$(libpq_srcdir) +Index: postgresql-9.5.0/contrib/postgres_fdw/deparse.c +=================================================================== +--- postgresql-9.5.0.orig/contrib/postgres_fdw/deparse.c 2016-01-20 14:40:31.000000000 +0000 ++++ postgresql-9.5.0/contrib/postgres_fdw/deparse.c 2016-01-20 14:50:42.518332653 +0000 +@@ -38,7 +38,6 @@ + #include "access/heapam.h" + #include "access/htup_details.h" + #include "access/sysattr.h" +-#include "access/transam.h" + #include "catalog/pg_collation.h" + #include "catalog/pg_namespace.h" + #include "catalog/pg_operator.h" +@@ -102,7 +101,7 @@ + static bool foreign_expr_walker(Node *node, + foreign_glob_cxt *glob_cxt, + foreign_loc_cxt *outer_cxt); +-static bool is_builtin(Oid procid); ++static char *deparse_type_name(Oid type_oid, int32 typemod); + + /* + * Functions to construct string representation of a node tree. +@@ -217,11 +216,12 @@ + * In addition, *outer_cxt is updated with collation information. + * + * We must check that the expression contains only node types we can deparse, +- * that all types/functions/operators are safe to send (which we approximate +- * as being built-in), and that all collations used in the expression derive +- * from Vars of the foreign table. Because of the latter, the logic is +- * pretty close to assign_collations_walker() in parse_collate.c, though we +- * can assume here that the given expression is valid. ++ * that all types/functions/operators are safe to send (they are "shippable"), ++ * and that all collations used in the expression derive from Vars of the ++ * foreign table. Because of the latter, the logic is pretty close to ++ * assign_collations_walker() in parse_collate.c, though we can assume here ++ * that the given expression is valid. Note function mutability is not ++ * currently considered here. + */ + static bool + foreign_expr_walker(Node *node, +@@ -229,6 +229,7 @@ + foreign_loc_cxt *outer_cxt) + { + bool check_type = true; ++ PgFdwRelationInfo *fpinfo; + foreign_loc_cxt inner_cxt; + Oid collation; + FDWCollateState state; +@@ -237,6 +238,9 @@ + if (node == NULL) + return true; + ++ /* May need server info from baserel's fdw_private struct */ ++ fpinfo = (PgFdwRelationInfo *) (glob_cxt->foreignrel->fdw_private); ++ + /* Set up inner_cxt for possible recursion to child nodes */ + inner_cxt.collation = InvalidOid; + inner_cxt.state = FDW_COLLATE_NONE; +@@ -374,11 +378,11 @@ + FuncExpr *fe = (FuncExpr *) node; + + /* +- * If function used by the expression is not built-in, it ++ * If function used by the expression is not shippable, it + * can't be sent to remote because it might have incompatible + * semantics on remote side. + */ +- if (!is_builtin(fe->funcid)) ++ if (!is_shippable(fe->funcid, ProcedureRelationId, fpinfo)) + return false; + + /* +@@ -422,11 +426,11 @@ + OpExpr *oe = (OpExpr *) node; + + /* +- * Similarly, only built-in operators can be sent to remote. +- * (If the operator is, surely its underlying function is +- * too.) ++ * Similarly, only shippable operators can be sent to remote. ++ * (If the operator is shippable, we assume its underlying ++ * function is too.) + */ +- if (!is_builtin(oe->opno)) ++ if (!is_shippable(oe->opno, OperatorRelationId, fpinfo)) + return false; + + /* +@@ -464,9 +468,9 @@ + ScalarArrayOpExpr *oe = (ScalarArrayOpExpr *) node; + + /* +- * Again, only built-in operators can be sent to remote. ++ * Again, only shippable operators can be sent to remote. + */ +- if (!is_builtin(oe->opno)) ++ if (!is_shippable(oe->opno, OperatorRelationId, fpinfo)) + return false; + + /* +@@ -613,10 +617,10 @@ + } + + /* +- * If result type of given expression is not built-in, it can't be sent to +- * remote because it might have incompatible semantics on remote side. ++ * If result type of given expression is not shippable, it can't be sent ++ * to remote because it might have incompatible semantics on remote side. + */ +- if (check_type && !is_builtin(exprType(node))) ++ if (check_type && !is_shippable(exprType(node), TypeRelationId, fpinfo)) + return false; + + /* +@@ -669,27 +673,23 @@ + } + + /* +- * Return true if given object is one of PostgreSQL's built-in objects. +- * +- * We use FirstBootstrapObjectId as the cutoff, so that we only consider +- * objects with hand-assigned OIDs to be "built in", not for instance any +- * function or type defined in the information_schema. +- * +- * Our constraints for dealing with types are tighter than they are for +- * functions or operators: we want to accept only types that are in pg_catalog, +- * else format_type might incorrectly fail to schema-qualify their names. +- * (This could be fixed with some changes to format_type, but for now there's +- * no need.) Thus we must exclude information_schema types. +- * +- * XXX there is a problem with this, which is that the set of built-in +- * objects expands over time. Something that is built-in to us might not +- * be known to the remote server, if it's of an older version. But keeping +- * track of that would be a huge exercise. ++ * Convert type OID + typmod info into a type name we can ship to the remote ++ * server. Someplace else had better have verified that this type name is ++ * expected to be known on the remote end. ++ * ++ * This is almost just format_type_with_typemod(), except that if left to its ++ * own devices, that function will make schema-qualification decisions based ++ * on the local search_path, which is wrong. We must schema-qualify all ++ * type names that are not in pg_catalog. We assume here that built-in types ++ * are all in pg_catalog and need not be qualified; otherwise, qualify. + */ +-static bool +-is_builtin(Oid oid) ++static char * ++deparse_type_name(Oid type_oid, int32 typemod) + { +- return (oid < FirstBootstrapObjectId); ++ if (is_builtin(type_oid)) ++ return format_type_with_typemod(type_oid, typemod); ++ else ++ return format_type_with_typemod_qualified(type_oid, typemod); + } + + +@@ -1355,8 +1355,8 @@ + { + appendStringInfoString(buf, "NULL"); + appendStringInfo(buf, "::%s", +- format_type_with_typemod(node->consttype, +- node->consttypmod)); ++ deparse_type_name(node->consttype, ++ node->consttypmod)); + return; + } + +@@ -1429,8 +1429,8 @@ + } + if (needlabel) + appendStringInfo(buf, "::%s", +- format_type_with_typemod(node->consttype, +- node->consttypmod)); ++ deparse_type_name(node->consttype, ++ node->consttypmod)); + } + + /* +@@ -1555,7 +1555,7 @@ + + deparseExpr((Expr *) linitial(node->args), context); + appendStringInfo(buf, "::%s", +- format_type_with_typemod(rettype, coercedTypmod)); ++ deparse_type_name(rettype, coercedTypmod)); + return; + } + +@@ -1750,8 +1750,8 @@ + deparseExpr(node->arg, context); + if (node->relabelformat != COERCE_IMPLICIT_CAST) + appendStringInfo(context->buf, "::%s", +- format_type_with_typemod(node->resulttype, +- node->resulttypmod)); ++ deparse_type_name(node->resulttype, ++ node->resulttypmod)); + } + + /* +@@ -1831,7 +1831,7 @@ + /* If the array is empty, we need an explicit cast to the array type. */ + if (node->elements == NIL) + appendStringInfo(buf, "::%s", +- format_type_with_typemod(node->array_typeid, -1)); ++ deparse_type_name(node->array_typeid, -1)); + } + + /* +@@ -1847,7 +1847,7 @@ + deparse_expr_cxt *context) + { + StringInfo buf = context->buf; +- char *ptypename = format_type_with_typemod(paramtype, paramtypmod); ++ char *ptypename = deparse_type_name(paramtype, paramtypmod); + + appendStringInfo(buf, "$%d::%s", paramindex, ptypename); + } +@@ -1873,7 +1873,7 @@ + deparse_expr_cxt *context) + { + StringInfo buf = context->buf; +- char *ptypename = format_type_with_typemod(paramtype, paramtypmod); ++ char *ptypename = deparse_type_name(paramtype, paramtypmod); + + appendStringInfo(buf, "((SELECT null::%s)::%s)", ptypename, ptypename); + } +Index: postgresql-9.5.0/contrib/postgres_fdw/option.c +=================================================================== +--- postgresql-9.5.0.orig/contrib/postgres_fdw/option.c 2016-01-20 14:40:31.000000000 +0000 ++++ postgresql-9.5.0/contrib/postgres_fdw/option.c 2016-01-20 14:50:42.518332653 +0000 +@@ -19,6 +19,8 @@ + #include "catalog/pg_foreign_table.h" + #include "catalog/pg_user_mapping.h" + #include "commands/defrem.h" ++#include "commands/extension.h" ++#include "utils/builtins.h" + + + /* +@@ -124,6 +126,11 @@ + errmsg("%s requires a non-negative numeric value", + def->defname))); + } ++ else if (strcmp(def->defname, "extensions") == 0) ++ { ++ /* check list syntax, warn about uninstalled extensions */ ++ (void) ExtractExtensionList(defGetString(def), true); ++ } + } + + PG_RETURN_VOID(); +@@ -150,6 +157,8 @@ + /* cost factors */ + {"fdw_startup_cost", ForeignServerRelationId, false}, + {"fdw_tuple_cost", ForeignServerRelationId, false}, ++ /* shippable extensions */ ++ {"extensions", ForeignServerRelationId, false}, + /* updatable is available on both server and table */ + {"updatable", ForeignServerRelationId, false}, + {"updatable", ForeignTableRelationId, false}, +@@ -293,3 +302,48 @@ + } + return i; + } ++ ++/* ++ * Parse a comma-separated string and return a List of the OIDs of the ++ * extensions named in the string. If any names in the list cannot be ++ * found, report a warning if warnOnMissing is true, else just silently ++ * ignore them. ++ */ ++List * ++ExtractExtensionList(const char *extensionsString, bool warnOnMissing) ++{ ++ List *extensionOids = NIL; ++ List *extlist; ++ ListCell *lc; ++ ++ /* SplitIdentifierString scribbles on its input, so pstrdup first */ ++ if (!SplitIdentifierString(pstrdup(extensionsString), ',', &extlist)) ++ { ++ /* syntax error in name list */ ++ ereport(ERROR, ++ (errcode(ERRCODE_INVALID_PARAMETER_VALUE), ++ errmsg("parameter \"%s\" must be a list of extension names", ++ "extensions"))); ++ } ++ ++ foreach(lc, extlist) ++ { ++ const char *extension_name = (const char *) lfirst(lc); ++ Oid extension_oid = get_extension_oid(extension_name, true); ++ ++ if (OidIsValid(extension_oid)) ++ { ++ extensionOids = lappend_oid(extensionOids, extension_oid); ++ } ++ else if (warnOnMissing) ++ { ++ ereport(WARNING, ++ (errcode(ERRCODE_UNDEFINED_OBJECT), ++ errmsg("extension \"%s\" is not installed", ++ extension_name))); ++ } ++ } ++ ++ list_free(extlist); ++ return extensionOids; ++} +Index: postgresql-9.5.0/contrib/postgres_fdw/postgres_fdw.c +=================================================================== +--- postgresql-9.5.0.orig/contrib/postgres_fdw/postgres_fdw.c 2016-01-20 14:49:20.242941217 +0000 ++++ postgresql-9.5.0/contrib/postgres_fdw/postgres_fdw.c 2016-01-20 14:50:42.518332653 +0000 +@@ -51,40 +51,6 @@ + #define DEFAULT_FDW_TUPLE_COST 0.01 + + /* +- * FDW-specific planner information kept in RelOptInfo.fdw_private for a +- * foreign table. This information is collected by postgresGetForeignRelSize. +- */ +-typedef struct PgFdwRelationInfo +-{ +- /* baserestrictinfo clauses, broken down into safe and unsafe subsets. */ +- List *remote_conds; +- List *local_conds; +- +- /* Bitmap of attr numbers we need to fetch from the remote server. */ +- Bitmapset *attrs_used; +- +- /* Cost and selectivity of local_conds. */ +- QualCost local_conds_cost; +- Selectivity local_conds_sel; +- +- /* Estimated size and cost for a scan with baserestrictinfo quals. */ +- double rows; +- int width; +- Cost startup_cost; +- Cost total_cost; +- +- /* Options extracted from catalogs. */ +- bool use_remote_estimate; +- Cost fdw_startup_cost; +- Cost fdw_tuple_cost; +- +- /* Cached catalog information. */ +- ForeignTable *table; +- ForeignServer *server; +- UserMapping *user; /* only set in use_remote_estimate mode */ +-} PgFdwRelationInfo; +- +-/* + * Indexes of FDW-private information stored in fdw_private lists. + * + * We store various information in ForeignScan.fdw_private to pass it from +@@ -412,6 +378,7 @@ + fpinfo->use_remote_estimate = false; + fpinfo->fdw_startup_cost = DEFAULT_FDW_STARTUP_COST; + fpinfo->fdw_tuple_cost = DEFAULT_FDW_TUPLE_COST; ++ fpinfo->shippable_extensions = NIL; + + foreach(lc, fpinfo->server->options) + { +@@ -423,6 +390,9 @@ + fpinfo->fdw_startup_cost = strtod(defGetString(def), NULL); + else if (strcmp(def->defname, "fdw_tuple_cost") == 0) + fpinfo->fdw_tuple_cost = strtod(defGetString(def), NULL); ++ else if (strcmp(def->defname, "extensions") == 0) ++ fpinfo->shippable_extensions = ++ ExtractExtensionList(defGetString(def), false); + } + foreach(lc, fpinfo->table->options) + { +Index: postgresql-9.5.0/contrib/postgres_fdw/postgres_fdw.h +=================================================================== +--- postgresql-9.5.0.orig/contrib/postgres_fdw/postgres_fdw.h 2016-01-20 14:40:31.000000000 +0000 ++++ postgresql-9.5.0/contrib/postgres_fdw/postgres_fdw.h 2016-01-20 14:50:42.518332653 +0000 +@@ -20,6 +20,41 @@ + + #include "libpq-fe.h" + ++/* ++ * FDW-specific planner information kept in RelOptInfo.fdw_private for a ++ * foreign table. This information is collected by postgresGetForeignRelSize. ++ */ ++typedef struct PgFdwRelationInfo ++{ ++ /* baserestrictinfo clauses, broken down into safe and unsafe subsets. */ ++ List *remote_conds; ++ List *local_conds; ++ ++ /* Bitmap of attr numbers we need to fetch from the remote server. */ ++ Bitmapset *attrs_used; ++ ++ /* Cost and selectivity of local_conds. */ ++ QualCost local_conds_cost; ++ Selectivity local_conds_sel; ++ ++ /* Estimated size and cost for a scan with baserestrictinfo quals. */ ++ double rows; ++ int width; ++ Cost startup_cost; ++ Cost total_cost; ++ ++ /* Options extracted from catalogs. */ ++ bool use_remote_estimate; ++ Cost fdw_startup_cost; ++ Cost fdw_tuple_cost; ++ List *shippable_extensions; /* OIDs of whitelisted extensions */ ++ ++ /* Cached catalog information. */ ++ ForeignTable *table; ++ ForeignServer *server; ++ UserMapping *user; /* only set in use_remote_estimate mode */ ++} PgFdwRelationInfo; ++ + /* in postgres_fdw.c */ + extern int set_transmission_modes(void); + extern void reset_transmission_modes(int nestlevel); +@@ -37,6 +72,8 @@ + extern int ExtractConnectionOptions(List *defelems, + const char **keywords, + const char **values); ++extern List *ExtractExtensionList(const char *extensionsString, ++ bool warnOnMissing); + + /* in deparse.c */ + extern void classifyConditions(PlannerInfo *root, +@@ -75,4 +112,8 @@ + List **retrieved_attrs); + extern void deparseStringLiteral(StringInfo buf, const char *val); + ++/* in shippable.c */ ++extern bool is_builtin(Oid objectId); ++extern bool is_shippable(Oid objectId, Oid classId, PgFdwRelationInfo *fpinfo); ++ + #endif /* POSTGRES_FDW_H */ +Index: postgresql-9.5.0/contrib/postgres_fdw/shippable.c +=================================================================== +--- /dev/null 1970-01-01 00:00:00.000000000 +0000 ++++ postgresql-9.5.0/contrib/postgres_fdw/shippable.c 2016-01-20 14:50:42.518332653 +0000 +@@ -0,0 +1,214 @@ ++/*------------------------------------------------------------------------- ++ * ++ * shippable.c ++ * Determine which database objects are shippable to a remote server. ++ * ++ * We need to determine whether particular functions, operators, and indeed ++ * data types are shippable to a remote server for execution --- that is, ++ * do they exist and have the same behavior remotely as they do locally? ++ * Built-in objects are generally considered shippable. Other objects can ++ * be shipped if they are white-listed by the user. ++ * ++ * Note: there are additional filter rules that prevent shipping mutable ++ * functions or functions using nonportable collations. Those considerations ++ * need not be accounted for here. ++ * ++ * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group ++ * ++ * IDENTIFICATION ++ * contrib/postgres_fdw/shippable.c ++ * ++ *------------------------------------------------------------------------- ++ */ ++ ++#include "postgres.h" ++ ++#include "postgres_fdw.h" ++ ++#include "access/transam.h" ++#include "catalog/dependency.h" ++#include "utils/hsearch.h" ++#include "utils/inval.h" ++#include "utils/syscache.h" ++ ++ ++/* Hash table for caching the results of shippability lookups */ ++static HTAB *ShippableCacheHash = NULL; ++ ++/* ++ * Hash key for shippability lookups. We include the FDW server OID because ++ * decisions may differ per-server. Otherwise, objects are identified by ++ * their (local!) OID and catalog OID. ++ */ ++typedef struct ++{ ++ /* XXX we assume this struct contains no padding bytes */ ++ Oid objid; /* function/operator/type OID */ ++ Oid classid; /* OID of its catalog (pg_proc, etc) */ ++ Oid serverid; /* FDW server we are concerned with */ ++} ShippableCacheKey; ++ ++typedef struct ++{ ++ ShippableCacheKey key; /* hash key - must be first */ ++ bool shippable; ++} ShippableCacheEntry; ++ ++ ++/* ++ * Flush cache entries when pg_foreign_server is updated. ++ * ++ * We do this because of the possibility of ALTER SERVER being used to change ++ * a server's extensions option. We do not currently bother to check whether ++ * objects' extension membership changes once a shippability decision has been ++ * made for them, however. ++ */ ++static void ++InvalidateShippableCacheCallback(Datum arg, int cacheid, uint32 hashvalue) ++{ ++ HASH_SEQ_STATUS status; ++ ShippableCacheEntry *entry; ++ ++ /* ++ * In principle we could flush only cache entries relating to the ++ * pg_foreign_server entry being outdated; but that would be more ++ * complicated, and it's probably not worth the trouble. So for now, just ++ * flush all entries. ++ */ ++ hash_seq_init(&status, ShippableCacheHash); ++ while ((entry = (ShippableCacheEntry *) hash_seq_search(&status)) != NULL) ++ { ++ if (hash_search(ShippableCacheHash, ++ (void *) &entry->key, ++ HASH_REMOVE, ++ NULL) == NULL) ++ elog(ERROR, "hash table corrupted"); ++ } ++} ++ ++/* ++ * Initialize the backend-lifespan cache of shippability decisions. ++ */ ++static void ++InitializeShippableCache(void) ++{ ++ HASHCTL ctl; ++ ++ /* Create the hash table. */ ++ MemSet(&ctl, 0, sizeof(ctl)); ++ ctl.keysize = sizeof(ShippableCacheKey); ++ ctl.entrysize = sizeof(ShippableCacheEntry); ++ ShippableCacheHash = ++ hash_create("Shippability cache", 256, &ctl, HASH_ELEM | HASH_BLOBS); ++ ++ /* Set up invalidation callback on pg_foreign_server. */ ++ CacheRegisterSyscacheCallback(FOREIGNSERVEROID, ++ InvalidateShippableCacheCallback, ++ (Datum) 0); ++} ++ ++/* ++ * Returns true if given object (operator/function/type) is shippable ++ * according to the server options. ++ * ++ * Right now "shippability" is exclusively a function of whether the object ++ * belongs to an extension declared by the user. In the future we could ++ * additionally have a whitelist of functions/operators declared one at a time. ++ */ ++static bool ++lookup_shippable(Oid objectId, Oid classId, PgFdwRelationInfo *fpinfo) ++{ ++ Oid extensionOid; ++ ++ /* ++ * Is object a member of some extension? (Note: this is a fairly ++ * expensive lookup, which is why we try to cache the results.) ++ */ ++ extensionOid = getExtensionOfObject(classId, objectId); ++ ++ /* If so, is that extension in fpinfo->shippable_extensions? */ ++ if (OidIsValid(extensionOid) && ++ list_member_oid(fpinfo->shippable_extensions, extensionOid)) ++ return true; ++ ++ return false; ++} ++ ++/* ++ * Return true if given object is one of PostgreSQL's built-in objects. ++ * ++ * We use FirstBootstrapObjectId as the cutoff, so that we only consider ++ * objects with hand-assigned OIDs to be "built in", not for instance any ++ * function or type defined in the information_schema. ++ * ++ * Our constraints for dealing with types are tighter than they are for ++ * functions or operators: we want to accept only types that are in pg_catalog, ++ * else deparse_type_name might incorrectly fail to schema-qualify their names. ++ * Thus we must exclude information_schema types. ++ * ++ * XXX there is a problem with this, which is that the set of built-in ++ * objects expands over time. Something that is built-in to us might not ++ * be known to the remote server, if it's of an older version. But keeping ++ * track of that would be a huge exercise. ++ */ ++bool ++is_builtin(Oid objectId) ++{ ++ return (objectId < FirstBootstrapObjectId); ++} ++ ++/* ++ * is_shippable ++ * Is this object (function/operator/type) shippable to foreign server? ++ */ ++bool ++is_shippable(Oid objectId, Oid classId, PgFdwRelationInfo *fpinfo) ++{ ++ ShippableCacheKey key; ++ ShippableCacheEntry *entry; ++ ++ /* Built-in objects are presumed shippable. */ ++ if (is_builtin(objectId)) ++ return true; ++ ++ /* Otherwise, give up if user hasn't specified any shippable extensions. */ ++ if (fpinfo->shippable_extensions == NIL) ++ return false; ++ ++ /* Initialize cache if first time through. */ ++ if (!ShippableCacheHash) ++ InitializeShippableCache(); ++ ++ /* Set up cache hash key */ ++ key.objid = objectId; ++ key.classid = classId; ++ key.serverid = fpinfo->server->serverid; ++ ++ /* See if we already cached the result. */ ++ entry = (ShippableCacheEntry *) ++ hash_search(ShippableCacheHash, ++ (void *) &key, ++ HASH_FIND, ++ NULL); ++ ++ if (!entry) ++ { ++ /* Not found in cache, so perform shippability lookup. */ ++ bool shippable = lookup_shippable(objectId, classId, fpinfo); ++ ++ /* ++ * Don't create a new hash entry until *after* we have the shippable ++ * result in hand, as the underlying catalog lookups might trigger a ++ * cache invalidation. ++ */ ++ entry = (ShippableCacheEntry *) ++ hash_search(ShippableCacheHash, ++ (void *) &key, ++ HASH_ENTER, ++ NULL); ++ ++ entry->shippable = shippable; ++ } ++ ++ return entry->shippable; ++} +Index: postgresql-9.5.0/doc/src/sgml/postgres-fdw.sgml +=================================================================== +--- postgresql-9.5.0.orig/doc/src/sgml/postgres-fdw.sgml 2016-01-20 14:40:31.000000000 +0000 ++++ postgresql-9.5.0/doc/src/sgml/postgres-fdw.sgml 2016-01-20 14:50:42.518332653 +0000 +@@ -265,6 +265,46 @@ + + + ++ Remote Execution Options ++ ++ ++ By default, only WHERE clauses using built-in operators and ++ functions will be considered for execution on the remote server. Clauses ++ involving non-built-in functions are checked locally after rows are ++ fetched. If such functions are available on the remote server and can be ++ relied on to produce the same results as they do locally, performance can ++ be improved by sending such WHERE clauses for remote ++ execution. This behavior can be controlled using the following option: ++ ++ ++ ++ ++ ++ extensions ++ ++ ++ This option is a comma-separated list of names ++ of PostgreSQL extensions that are installed, in ++ compatible versions, on both the local and remote servers. Functions ++ and operators that are immutable and belong to a listed extension will ++ be considered shippable to the remote server. ++ This option can only be specified for foreign servers, not per-table. ++ ++ ++ ++ ++ ++ ++ ++ When using the extensions option, it is the ++ user's responsibility that the listed extensions exist and behave ++ identically on both the local and remote servers. Otherwise, remote ++ queries may fail or behave unexpectedly. ++ ++ ++ ++ ++ + Updatability Options + + +@@ -427,8 +467,10 @@ + execution, and by not retrieving table columns that are not needed for + the current query. To reduce the risk of misexecution of queries, + WHERE clauses are not sent to the remote server unless they use +- only built-in data types, operators, and functions. Operators and +- functions in the clauses must be IMMUTABLE as well. ++ only data types, operators, and functions that are built-in or belong to an ++ extension that's listed in the foreign server's extensions ++ option. Operators and functions in such clauses must ++ be IMMUTABLE as well. + + + +Index: postgresql-9.5.0/src/backend/utils/adt/format_type.c +=================================================================== +--- postgresql-9.5.0.orig/src/backend/utils/adt/format_type.c 2016-01-20 14:40:31.000000000 +0000 ++++ postgresql-9.5.0/src/backend/utils/adt/format_type.c 2016-01-20 14:50:42.522332772 +0000 +@@ -97,7 +97,8 @@ + } + + /* +- * This version returns a name which is always qualified. ++ * This version returns a name that is always qualified (unless it's one ++ * of the SQL-keyword type names, such as TIMESTAMP WITH TIME ZONE). + */ + char * + format_type_be_qualified(Oid type_oid) +@@ -114,6 +115,19 @@ + return format_type_internal(type_oid, typemod, true, false, false); + } + ++/* ++ * This version allows a nondefault typemod to be specified, and forces ++ * qualification of normal type names. ++ */ ++char * ++format_type_with_typemod_qualified(Oid type_oid, int32 typemod) ++{ ++ return format_type_internal(type_oid, typemod, true, false, true); ++} ++ ++/* ++ * Common workhorse. ++ */ + static char * + format_type_internal(Oid type_oid, int32 typemod, + bool typemod_given, bool allow_invalid, +Index: postgresql-9.5.0/src/include/utils/builtins.h +=================================================================== +--- postgresql-9.5.0.orig/src/include/utils/builtins.h 2016-01-20 14:40:31.000000000 +0000 ++++ postgresql-9.5.0/src/include/utils/builtins.h 2016-01-20 14:50:42.522332772 +0000 +@@ -1105,6 +1105,7 @@ + extern char *format_type_be(Oid type_oid); + extern char *format_type_be_qualified(Oid type_oid); + extern char *format_type_with_typemod(Oid type_oid, int32 typemod); ++extern char *format_type_with_typemod_qualified(Oid type_oid, int32 typemod); + extern Datum oidvectortypes(PG_FUNCTION_ARGS); + extern int32 type_maximum_size(Oid type_oid, int32 typemod); + diff --git a/debian/patches/series b/debian/patches/series index b5908f8..cfac1fd 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -7,3 +7,5 @@ 64-pg_upgrade-sockdir 70-history 80-reproducible +90-profile_fdw.patch +91-extension_shipping.patch From f072142b7bb2d71d1c644288d030f16600ab6005 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 26 Jan 2016 12:05:13 +0100 Subject: [PATCH 02/31] Move spinlock flag to common configure flags, the python3 build needs that as well. --- debian/changelog | 7 +++++++ debian/rules | 12 ++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/debian/changelog b/debian/changelog index 5ef21ff..a783ab2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +postgresql-9.5 (9.5.0-3) UNRELEASED; urgency=medium + + * Move spinlock flag to common configure flags, the python3 build needs that + as well. + + -- Christoph Berg Tue, 26 Jan 2016 12:04:06 +0100 + postgresql-9.5 (9.5.0-2) unstable; urgency=medium * Disable spinlocks on alpha. Performance will be poor, but at least they diff --git a/debian/rules b/debian/rules index bd379f8..e79c4e6 100755 --- a/debian/rules +++ b/debian/rules @@ -39,6 +39,11 @@ endif export DPKG_GENSYMBOLS_CHECK_LEVEL=4 +# PostgreSQL 9.5+ does not have native spinlock support on this platform +ifneq ($(findstring $(DEB_BUILD_ARCH), alpha),) +SPINLOCK_FLAGS = --disable-spinlocks +endif + COMMON_CONFIGURE_FLAGS= \ --mandir=/usr/share/postgresql/$(MAJOR_VER)/man \ --docdir=/usr/share/doc/postgresql-doc-$(MAJOR_VER) \ @@ -59,6 +64,7 @@ COMMON_CONFIGURE_FLAGS= \ --with-gnu-ld \ --with-pgport=5432 \ --with-system-tzdata=/usr/share/zoneinfo \ + $(SPINLOCK_FLAGS) \ CFLAGS='$(CFLAGS)' \ LDFLAGS='$(LDFLAGS)' @@ -69,11 +75,6 @@ else BOOTSTRAP_FLAGS= --with-krb5 --with-gssapi --with-ldap endif -# PostgreSQL 9.5+ does not have native spinlock support on this platform -ifneq ($(findstring $(DEB_BUILD_ARCH), alpha),) -SPINLOCK_FLAGS = --disable-spinlocks -endif - # --with-selinux needs libselinux 2.1.10 # (we don't put that in debian/control so we can use the same source also on older dists) SEVERSION = $(shell dpkg-query -f '$${Version}' --show libselinux1-dev) @@ -111,7 +112,6 @@ stamp/configure-build: PYTHON=/usr/bin/python \ $(COMMON_CONFIGURE_FLAGS) \ $(BOOTSTRAP_FLAGS) \ - $(SPINLOCK_FLAGS) \ $(SELINUX_FLAGS) touch "$@" From 00112df1d529037e450799941c1accd60b90d382 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Thu, 4 Feb 2016 13:42:22 +0100 Subject: [PATCH 03/31] Remove preinst, the catversion change logic is in postgresql-common now --- debian/postgresql-9.5.preinst | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 debian/postgresql-9.5.preinst diff --git a/debian/postgresql-9.5.preinst b/debian/postgresql-9.5.preinst deleted file mode 100644 index c5bfca1..0000000 --- a/debian/postgresql-9.5.preinst +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -e - -## DB format changed between beta 1 and 2 -#if [ "$1" = "upgrade" ] || [ "$1" = "install" ] && \ -# dpkg --compare-versions "$2" lt-nl "9.4~beta2"; then -# if pg_lsclusters -h | grep -q '^9.4'; then -# echo "ERROR: The database format changed between beta 1 and 2. Please dump your 9.4 clusters first and remove them, before attempting to upgrade the package." >&2 -# exit 1 -# fi -#fi - -#DEBHELPER# - -exit 0 From 28b726f80b3ab8f5c66a86c262a99de0eee96d5a Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Thu, 4 Feb 2016 13:43:54 +0100 Subject: [PATCH 04/31] Update debian/copyright to please DEP-5. --- debian/changelog | 5 +++-- debian/copyright | 11 ++++++----- debian/source/lintian-overrides | 3 +++ 3 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 debian/source/lintian-overrides diff --git a/debian/changelog b/debian/changelog index a783ab2..a9354b3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,9 +1,10 @@ -postgresql-9.5 (9.5.0-3) UNRELEASED; urgency=medium +postgresql-9.5 (9.5.0-3) unstable; urgency=medium * Move spinlock flag to common configure flags, the python3 build needs that as well. + * Update debian/copyright to please DEP-5. - -- Christoph Berg Tue, 26 Jan 2016 12:04:06 +0100 + -- Christoph Berg Thu, 04 Feb 2016 13:08:13 +0100 postgresql-9.5 (9.5.0-2) unstable; urgency=medium diff --git a/debian/copyright b/debian/copyright index f93846d..896cd6f 100644 --- a/debian/copyright +++ b/debian/copyright @@ -5,7 +5,7 @@ Source: ftp://ftp.postgresql.org/pub/source/ Files: * Copyright: Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California -License: +License: PostgreSQL Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this @@ -25,7 +25,7 @@ License: Files: src/backend/regex/* Copyright: Copyright (c) 1998, 1999 Henry Spencer. All rights reserved. -License: +License: Custom-regex Development of this software was funded, in part, by Cray Research Inc., UUNET Communications Services Inc., Sun Microsystems Inc., and Scriptics Corporation, none of whom are responsible for the results. The author @@ -61,7 +61,7 @@ License: Tcl Files: src/bin/pg_dump/* Copyright: Portions Copyright (c) 2000, Philip Warner -License: +License: Custom-pg_dump Rights are granted to use this software in any way so long as this notice is not removed. The author is not responsible for loss or damages that may result from its use. @@ -70,13 +70,14 @@ Files: src/backend/utils/mb/Unicode/*.txt Copyright: Copyright (C) 2001 earthian@tama.or.jp, All Rights Reserved. Copyright (C) 2001 I'O, All Rights Reserved. Copyright (C) 2006 Project X0213, All Rights Reserved. -License: +License: Custom-Unicode You can use, modify, distribute this table freely. Files: contrib/* -License: almost exclusively BSD +Copyright: Copyright details for contrib modules are detailed in a separate copyright file in the postgresql-contrib package. +License: almost exclusively BSD License: Tcl This software is copyrighted by the Regents of the University of diff --git a/debian/source/lintian-overrides b/debian/source/lintian-overrides new file mode 100644 index 0000000..13da7ef --- /dev/null +++ b/debian/source/lintian-overrides @@ -0,0 +1,3 @@ +# The actual license is in a different file +postgresql-9.5 source: space-in-std-shortname-in-dep5-copyright almost exclusively bsd* +postgresql-9.5 source: missing-license-paragraph-in-dep5-copyright almost exclusively bsd* From 5b6e0e5c1ecfb2c493def0defb8d16894204c512 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Wed, 10 Feb 2016 13:56:21 +0100 Subject: [PATCH 05/31] New upstream version. * New upstream version. + Fix infinite loops and buffer-overrun problems in regular expressions. Very large character ranges in bracket expressions could cause infinite loops in some cases, and memory overwrites in other cases. (CVE-2016-0773) * Fix installation path of contrib examples. --- debian/changelog | 12 ++++++++++++ debian/patches/80-reproducible | 11 ----------- debian/patches/series | 1 - debian/postgresql-contrib-9.5.install | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) delete mode 100644 debian/patches/80-reproducible diff --git a/debian/changelog b/debian/changelog index a9354b3..b214139 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +postgresql-9.5 (9.5.1-1) unstable; urgency=medium + + * New upstream version. + + Fix infinite loops and buffer-overrun problems in regular expressions. + Very large character ranges in bracket expressions could cause infinite + loops in some cases, and memory overwrites in other cases. + (CVE-2016-0773) + + * Fix installation path of contrib examples. + + -- Christoph Berg Wed, 10 Feb 2016 13:04:39 +0100 + postgresql-9.5 (9.5.0-3) unstable; urgency=medium * Move spinlock flag to common configure flags, the python3 build needs that diff --git a/debian/patches/80-reproducible b/debian/patches/80-reproducible deleted file mode 100644 index 9d9de4a..0000000 --- a/debian/patches/80-reproducible +++ /dev/null @@ -1,11 +0,0 @@ ---- a/src/bin/pg_xlogdump/Makefile -+++ b/src/bin/pg_xlogdump/Makefile -@@ -12,7 +12,7 @@ OBJS = pg_xlogdump.o compat.o xlogreader - - override CPPFLAGS := -DFRONTEND $(CPPFLAGS) - --RMGRDESCSOURCES = $(notdir $(wildcard $(top_srcdir)/src/backend/access/rmgrdesc/*desc.c)) -+RMGRDESCSOURCES = $(sort $(notdir $(wildcard $(top_srcdir)/src/backend/access/rmgrdesc/*desc.c))) - RMGRDESCOBJS = $(patsubst %.c,%.o,$(RMGRDESCSOURCES)) - - diff --git a/debian/patches/series b/debian/patches/series index b5908f8..562acd0 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -6,4 +6,3 @@ 54-debian-alternatives-for-external-tools.patch 64-pg_upgrade-sockdir 70-history -80-reproducible diff --git a/debian/postgresql-contrib-9.5.install b/debian/postgresql-contrib-9.5.install index e49a323..c8bb87b 100644 --- a/debian/postgresql-contrib-9.5.install +++ b/debian/postgresql-contrib-9.5.install @@ -1,4 +1,4 @@ -usr/share/doc/postgresql-doc-*/extension/* usr/share/doc/postgresql-contrib-* +usr/share/doc/postgresql-doc-*/extension/* usr/share/doc/postgresql-contrib-9.5 usr/share/postgresql/*/extension/* usr/lib/postgresql/*/lib/auth_delay.so usr/lib/postgresql/*/bin/pg_archivecleanup From 0351cd024cbd074e48f05e6f55d646782fdaad18 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Thu, 11 Feb 2016 22:43:35 +0100 Subject: [PATCH 06/31] Stop suggesting the use of identd. --- debian/changelog | 6 ++++++ debian/control | 6 ++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index b214139..700711f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +postgresql-9.5 (9.5.1-2) UNRELEASED; urgency=medium + + * Stop suggesting the use of identd. + + -- Christoph Berg Thu, 11 Feb 2016 22:41:48 +0100 + postgresql-9.5 (9.5.1-1) unstable; urgency=medium * New upstream version. diff --git a/debian/control b/debian/control index a88d9c7..14651a5 100644 --- a/debian/control +++ b/debian/control @@ -137,7 +137,7 @@ Depends: ${misc:Depends}, ssl-cert, locales Recommends: postgresql-contrib-9.5 -Suggests: oidentd | ident-server, locales-all +Suggests: locales-all Description: object-relational SQL database, version 9.5 server PostgreSQL is a fully featured object-relational database management system. It supports a large part of the SQL standard and is designed @@ -149,9 +149,7 @@ Description: object-relational SQL database, version 9.5 server . This package provides the database server for PostgreSQL 9.5. Servers for other major release versions can be installed simultaneously and - are coordinated by the postgresql-common package. A package providing - ident-server is needed if you want to authenticate remote connections - with identd. + are coordinated by the postgresql-common package. Package: postgresql-9.5-dbg Section: debug From 3248d11f4720340d2062b0f205bb827db11056af Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Thu, 11 Feb 2016 23:03:44 +0100 Subject: [PATCH 07/31] Modernize server package description. --- debian/changelog | 1 + debian/control | 30 ++++++++++++------------------ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/debian/changelog b/debian/changelog index 700711f..1bd47cf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ postgresql-9.5 (9.5.1-2) UNRELEASED; urgency=medium * Stop suggesting the use of identd. + * Modernize server package description. -- Christoph Berg Thu, 11 Feb 2016 22:41:48 +0100 diff --git a/debian/control b/debian/control index 14651a5..c4f1256 100644 --- a/debian/control +++ b/debian/control @@ -139,17 +139,17 @@ Depends: ${misc:Depends}, Recommends: postgresql-contrib-9.5 Suggests: locales-all Description: object-relational SQL database, version 9.5 server - PostgreSQL is a fully featured object-relational database management - system. It supports a large part of the SQL standard and is designed - to be extensible by users in many aspects. Some of the features are: - ACID transactions, foreign keys, views, sequences, subqueries, - triggers, user-defined types and functions, outer joins, multiversion - concurrency control. Graphical user interfaces and bindings for many - programming languages are available as well. + PostgreSQL is a powerful, open source object-relational database + system. It is fully ACID compliant, has full support for foreign + keys, joins, views, triggers, and stored procedures (in multiple + languages). It includes most SQL:2008 data types, including INTEGER, + NUMERIC, BOOLEAN, CHAR, VARCHAR, DATE, INTERVAL, and TIMESTAMP. It + also supports storage of binary large objects, including pictures, + sounds, or video. It has native programming interfaces for C/C++, + Java, .Net, Perl, Python, Ruby, Tcl, ODBC, among others, and + exceptional documentation. . - This package provides the database server for PostgreSQL 9.5. Servers - for other major release versions can be installed simultaneously and - are coordinated by the postgresql-common package. + This package provides the database server for PostgreSQL 9.5. Package: postgresql-9.5-dbg Section: debug @@ -157,15 +157,9 @@ Priority: extra Architecture: any Depends: ${misc:Depends}, postgresql-9.5 (= ${binary:Version}) Description: debug symbols for postgresql-9.5 - PostgreSQL is a fully featured object-relational database management - system. It supports a large part of the SQL standard and is designed - to be extensible by users in many aspects. Some of the features are: - ACID transactions, foreign keys, views, sequences, subqueries, - triggers, user-defined types and functions, outer joins, multiversion - concurrency control. Graphical user interfaces and bindings for many - programming languages are available as well. - . This package provides detached debugging symbols for PostgreSQL 9.5. + . + PostgreSQL is an object-relational SQL database management system. Package: postgresql-client-9.5 Architecture: any From 5b964c04eeda544d24c36fae94f4769e666ba885 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Thu, 11 Feb 2016 23:10:29 +0100 Subject: [PATCH 08/31] Recommend sysstat. --- debian/changelog | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 1bd47cf..151ae59 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,7 @@ postgresql-9.5 (9.5.1-2) UNRELEASED; urgency=medium * Stop suggesting the use of identd. * Modernize server package description. + * Recommend sysstat. -- Christoph Berg Thu, 11 Feb 2016 22:41:48 +0100 diff --git a/debian/control b/debian/control index c4f1256..58d6374 100644 --- a/debian/control +++ b/debian/control @@ -136,7 +136,7 @@ Depends: ${misc:Depends}, tzdata, ssl-cert, locales -Recommends: postgresql-contrib-9.5 +Recommends: postgresql-contrib-9.5, sysstat Suggests: locales-all Description: object-relational SQL database, version 9.5 server PostgreSQL is a powerful, open source object-relational database From 8598fd5ecb5e079b2975f987de04db9c2876c56c Mon Sep 17 00:00:00 2001 From: Antonio Zamorano Date: Thu, 3 Mar 2016 17:33:14 +0100 Subject: [PATCH 09/31] Removing patch that was wrongly added while resolving a conflict --- debian/patches/series | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/patches/series b/debian/patches/series index cfac1fd..348b49f 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -6,6 +6,5 @@ 54-debian-alternatives-for-external-tools.patch 64-pg_upgrade-sockdir 70-history -80-reproducible 90-profile_fdw.patch 91-extension_shipping.patch From a31c1f0274695e2afcbf07aab40b0e291b12ab0c Mon Sep 17 00:00:00 2001 From: Antonio Zamorano Date: Thu, 3 Mar 2016 17:33:37 +0100 Subject: [PATCH 10/31] Bumping version for CartoDB --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 546a401..ff05816 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +postgresql-9.5 (9.5.1-1cdb1) precise; urgency=medium + + * Release for CartoDB + + -- Antonio Zamorano Thu, 03 Mar 2016 15:56:13 +0100 + postgresql-9.5 (9.5.1-1) unstable; urgency=medium * New upstream version. From 17ede8875f48cdad7370c18b14f6be068bd9ac0b Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 29 Mar 2016 12:12:13 +0200 Subject: [PATCH 11/31] Bump S-V --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 58d6374..7a27474 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Section: database Priority: optional Maintainer: Debian PostgreSQL Maintainers Uploaders: Martin Pitt , Peter Eisentraut , Christoph Berg -Standards-Version: 3.9.6 +Standards-Version: 3.9.7 Build-Depends: debhelper (>= 9~), dpkg-dev (>= 1.16.1~) | hardening-wrapper, perl (>= 5.8), From adb40ce1752cf5265dcc1fdfe6ba49131ae44960 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 29 Mar 2016 12:33:01 +0200 Subject: [PATCH 12/31] New upstream version. + Disable abbreviated keys for string sorting in non-C locales (Robert Haas) PostgreSQL 9.5 introduced logic for speeding up comparisons of string data types by using the standard C library function strxfrm() as a substitute for strcoll(). It now emerges that most versions of glibc (Linux's implementation of the C library) have buggy implementations of strxfrm() that, in some locales, can produce string comparison results that do not match strcoll(). Until this problem can be better characterized, disable the optimization in all non-C locales. (C locale is safe since it uses neither strcoll() nor strxfrm().) Unfortunately, this problem affects not only sorting but also entry ordering in B-tree indexes, which means that B-tree indexes on text, varchar, or char columns may now be corrupt if they sort according to an affected locale and were built or modified under PostgreSQL 9.5.0 or 9.5.1. Users should REINDEX indexes that might be affected. It is not possible at this time to give an exhaustive list of known-affected locales. C locale is known safe, and there is no evidence of trouble in English-based locales such as en_US, but some other popular locales such as de_DE are affected in most glibc versions. + Maintain row-security status properly in cached plans (Stephen Frost) In a session that performs queries as more than one role, the plan cache might incorrectly re-use a plan that was generated for another role ID, thus possibly applying the wrong set of policies when row-level security (RLS) is in use. (CVE-2016-2193) + Add must-be-superuser checks to some new contrib/pageinspect functions (Andreas Seltenreich) Most functions in the pageinspect extension that inspect bytea values disallow calls by non-superusers, but brin_page_type() and brin_metapage_info() failed to do so. Passing contrived bytea values to them might crash the server or disclose a few bytes of server memory. Add the missing permissions checks to prevent misuse. (CVE-2016-3065) --- debian/changelog | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 151ae59..7a41025 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,10 +1,51 @@ -postgresql-9.5 (9.5.1-2) UNRELEASED; urgency=medium +postgresql-9.5 (9.5.2-1) unstable; urgency=medium + + * New upstream version. + + + Disable abbreviated keys for string sorting in non-C locales + (Robert Haas) + + PostgreSQL 9.5 introduced logic for speeding up comparisons of string + data types by using the standard C library function strxfrm() as a + substitute for strcoll(). It now emerges that most versions of glibc + (Linux's implementation of the C library) have buggy implementations of + strxfrm() that, in some locales, can produce string comparison results + that do not match strcoll(). Until this problem can be better + characterized, disable the optimization in all non-C locales. (C locale + is safe since it uses neither strcoll() nor strxfrm().) + + Unfortunately, this problem affects not only sorting but also entry + ordering in B-tree indexes, which means that B-tree indexes on text, + varchar, or char columns may now be corrupt if they sort according to an + affected locale and were built or modified under PostgreSQL 9.5.0 or + 9.5.1. Users should REINDEX indexes that might be affected. + + It is not possible at this time to give an exhaustive list of + known-affected locales. C locale is known safe, and there is no + evidence of trouble in English-based locales such as en_US, but some + other popular locales such as de_DE are affected in most glibc versions. + + + Maintain row-security status properly in cached plans (Stephen Frost) + + In a session that performs queries as more than one role, the plan cache + might incorrectly re-use a plan that was generated for another role ID, + thus possibly applying the wrong set of policies when row-level security + (RLS) is in use. (CVE-2016-2193) + + + Add must-be-superuser checks to some new contrib/pageinspect functions + (Andreas Seltenreich) + + Most functions in the pageinspect extension that inspect bytea values + disallow calls by non-superusers, but brin_page_type() and + brin_metapage_info() failed to do so. Passing contrived bytea values to + them might crash the server or disclose a few bytes of server memory. + Add the missing permissions checks to prevent misuse. (CVE-2016-3065) * Stop suggesting the use of identd. * Modernize server package description. * Recommend sysstat. - -- Christoph Berg Thu, 11 Feb 2016 22:41:48 +0100 + -- Christoph Berg Tue, 29 Mar 2016 12:22:08 +0200 postgresql-9.5 (9.5.1-1) unstable; urgency=medium From d55b07bdec19b2e7861ce8995860dd2d6362ba2e Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 29 Mar 2016 16:34:13 +0200 Subject: [PATCH 13/31] Add postgresql-9.5.NEWS noting the strxfrm() problem --- debian/postgresql-9.5.NEWS | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 debian/postgresql-9.5.NEWS diff --git a/debian/postgresql-9.5.NEWS b/debian/postgresql-9.5.NEWS new file mode 100644 index 0000000..607a025 --- /dev/null +++ b/debian/postgresql-9.5.NEWS @@ -0,0 +1,28 @@ +postgresql-9.5 (9.5.2-1) unstable; urgency=medium + + Users of PostgreSQL 9.5.0 or 9.5.1 clusters should note the following change: + + + Disable abbreviated keys for string sorting in non-C locales + (Robert Haas) + + PostgreSQL 9.5 introduced logic for speeding up comparisons of string + data types by using the standard C library function strxfrm() as a + substitute for strcoll(). It now emerges that most versions of glibc + (Linux's implementation of the C library) have buggy implementations of + strxfrm() that, in some locales, can produce string comparison results + that do not match strcoll(). Until this problem can be better + characterized, disable the optimization in all non-C locales. (C locale + is safe since it uses neither strcoll() nor strxfrm().) + + Unfortunately, this problem affects not only sorting but also entry + ordering in B-tree indexes, which means that B-tree indexes on text, + varchar, or char columns may now be corrupt if they sort according to an + affected locale and were built or modified under PostgreSQL 9.5.0 or + 9.5.1. Users should REINDEX indexes that might be affected. + + It is not possible at this time to give an exhaustive list of + known-affected locales. C locale is known safe, and there is no + evidence of trouble in English-based locales such as en_US, but some + other popular locales such as de_DE are affected in most glibc versions. + + -- Christoph Berg Tue, 29 Mar 2016 12:57:56 +0200 From d44bf8756bc6604ce1f6bc85c3413dd977914a31 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 29 Mar 2016 20:49:15 +0200 Subject: [PATCH 14/31] 02-relax-sslkey-permscheck.patch: Replace with what went upstream in 9.6. --- debian/changelog | 1 + .../patches/02-relax-sslkey-permscheck.patch | 96 ++++++++++++++++--- 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/debian/changelog b/debian/changelog index 7a41025..97431bf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -41,6 +41,7 @@ postgresql-9.5 (9.5.2-1) unstable; urgency=medium them might crash the server or disclose a few bytes of server memory. Add the missing permissions checks to prevent misuse. (CVE-2016-3065) + * 02-relax-sslkey-permscheck.patch: Replace with what went upstream in 9.6. * Stop suggesting the use of identd. * Modernize server package description. * Recommend sysstat. diff --git a/debian/patches/02-relax-sslkey-permscheck.patch b/debian/patches/02-relax-sslkey-permscheck.patch index 761f1ef..f07bd30 100644 --- a/debian/patches/02-relax-sslkey-permscheck.patch +++ b/debian/patches/02-relax-sslkey-permscheck.patch @@ -1,25 +1,97 @@ -Description: Relax security check on private SSL key file: The file is also considered safe if it has owner and/or group "root", and if it is group-readable (unless the group is something other than root or the database owner group). -Author: Martin Pitt -Forwarded: Yes, but rejected upstream -Bug-Debian: http://bugs.debian.org/3279.2 +commit 9a83564c58b7f6363141a8f1d0c87c89a5ebab5d +Author: Peter Eisentraut +Date: Sat Mar 19 11:03:22 2016 +0100 + Allow SSL server key file to have group read access if owned by root + + We used to require the server key file to have permissions 0600 or less + for best security. But some systems (such as Debian) have certificate + and key files managed by the operating system that can be shared with + other services. In those cases, the "postgres" user is made a member of + a special group that has access to those files, and the server key file + has permissions 0640. To accommodate that kind of setup, also allow the + key file to have permissions 0640 but only if owned by root. + + From: Christoph Berg + Reviewed-by: Alvaro Herrera + +diff --git a/doc/src/sgml/runtime.sgml b/doc/src/sgml/runtime.sgml +index c699f21..4a0e35a 100644 +--- a/doc/src/sgml/runtime.sgml ++++ b/doc/src/sgml/runtime.sgml +@@ -2147,9 +2147,20 @@ pg_dumpall -p 5432 | psql -d postgres -p 5433 + the server's data directory, but other names and locations can be specified + using the configuration parameters + and . ++ ++ ++ + On Unix systems, the permissions on server.key must + disallow any access to world or group; achieve this by the command +- chmod 0600 server.key. ++ chmod 0600 server.key. Alternatively, the file can be ++ owned by root and have group read access (that is, 0640 ++ permissions). That setup is intended for installations where certificate ++ and key files are managed by the operating system. The user under which ++ the PostgreSQL server runs should then be made a ++ member of the group that has access to those certificate and key files. ++ ++ ++ + If the private key is protected with a passphrase, the + server will prompt for the passphrase and will not start until it has + been entered. +diff --git a/src/backend/libpq/be-secure-openssl.c b/src/backend/libpq/be-secure-openssl.c +index 1e3dfb6..6009663 100644 --- a/src/backend/libpq/be-secure-openssl.c +++ b/src/backend/libpq/be-secure-openssl.c -@@ -687,12 +687,15 @@ be_tls_init(void) +@@ -206,8 +206,30 @@ be_tls_init(void) + errmsg("could not access private key file \"%s\": %m", + ssl_key_file))); + ++ if (!S_ISREG(buf.st_mode)) ++ ereport(FATAL, ++ (errcode(ERRCODE_CONFIG_FILE_ERROR), ++ errmsg("private key file \"%s\" is not a regular file", ++ ssl_key_file))); ++ ++ /* ++ * Refuse to load files owned by users other than us or root. ++ * ++ * XXX surely we can check this on Windows somehow, too. ++ */ ++#if !defined(WIN32) && !defined(__CYGWIN__) ++ if (buf.st_uid != geteuid() && buf.st_uid != 0) ++ ereport(FATAL, ++ (errcode(ERRCODE_CONFIG_FILE_ERROR), ++ errmsg("private key file \"%s\" must be owned by the database user or root", ++ ssl_key_file))); ++#endif ++ + /* +- * Require no public access to key file. ++ * Require no public access to key file. If the file is owned by us, ++ * require mode 0600 or less. If owned by root, require 0640 or less ++ * to allow read access through our gid, or a supplementary gid that ++ * allows to read system-wide certificates. + * + * XXX temporarily suppress check when on Windows, because there may + * not be proper support for Unix-y file permissions. Need to think +@@ -215,12 +237,13 @@ be_tls_init(void) * directory permission check in postmaster.c) */ #if !defined(WIN32) && !defined(__CYGWIN__) - if (!S_ISREG(buf.st_mode) || buf.st_mode & (S_IRWXG | S_IRWXO)) -+ if (!S_ISREG(buf.st_mode) || (buf.st_mode & (S_IWGRP | S_IRWXO)) || -+ ((buf.st_uid != geteuid()) && buf.st_uid != 0)) ++ if ((buf.st_uid == geteuid() && buf.st_mode & (S_IRWXG | S_IRWXO)) || ++ (buf.st_uid == 0 && buf.st_mode & (S_IWGRP | S_IXGRP | S_IRWXO))) ereport(FATAL, (errcode(ERRCODE_CONFIG_FILE_ERROR), - errmsg("private key file \"%s\" has group or world access", - ssl_key_file), +- errmsg("private key file \"%s\" has group or world access", +- ssl_key_file), - errdetail("Permissions should be u=rw (0600) or less."))); -+ errdetail("File must be owned by the \ -+database user or root, must have no write permission for \"group\", and must \ -+have no permissions for \"other\"."))); ++ errmsg("private key file \"%s\" has group or world access", ++ ssl_key_file), ++ errdetail("File must have permissions u=rw (0600) or less if owned by the database user, or permissions u=rw,g=r (0640) or less if owned by root."))); #endif if (SSL_CTX_use_PrivateKey_file(SSL_context, From a528db659cf4a4ea07471d2d542d445eca2f350b Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 12 Apr 2016 11:58:22 +0200 Subject: [PATCH 15/31] Use host architecture for all architecture checks --- debian/changelog | 6 ++++++ debian/rules | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index 97431bf..9553267 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +postgresql-9.5 (9.5.2-2) UNRELEASED; urgency=medium + + * rules: Use host architecture for all architecture checks. + + -- Christoph Berg Tue, 12 Apr 2016 11:59:21 +0200 + postgresql-9.5 (9.5.2-1) unstable; urgency=medium * New upstream version. diff --git a/debian/rules b/debian/rules index e79c4e6..28e1b0d 100755 --- a/debian/rules +++ b/debian/rules @@ -6,7 +6,7 @@ TCL_VER := 8.5 endif MAJOR_VER := 9.5 -DEB_BUILD_ARCH = $(shell dpkg-architecture -qDEB_BUILD_ARCH) +DEB_HOST_ARCH = $(shell dpkg-architecture -qDEB_HOST_ARCH) # this must also work for old releases with multiarch, so don't fail if the # variable doesn't exist @@ -33,14 +33,14 @@ CFLAGS+= -fPIC -pie endif # Facilitate hierarchical profile generation on amd64 (#730134) -ifeq ($(shell dpkg-architecture -qDEB_BUILD_ARCH),amd64) +ifeq ($(shell dpkg-architecture -qDEB_HOST_ARCH),amd64) CFLAGS+= -fno-omit-frame-pointer endif export DPKG_GENSYMBOLS_CHECK_LEVEL=4 # PostgreSQL 9.5+ does not have native spinlock support on this platform -ifneq ($(findstring $(DEB_BUILD_ARCH), alpha),) +ifneq ($(findstring $(DEB_HOST_ARCH), alpha),) SPINLOCK_FLAGS = --disable-spinlocks endif @@ -88,7 +88,7 @@ endif TESTSUITE_FAIL_CMD=exit 1 # hurd doesn't implement semaphores; succeed anyway so they at least have libpq5 # plperl currently fails on kfreebsd-* -ifneq ($(filter $(DEB_BUILD_ARCH), hurd-i386 kfreebsd-amd64 kfreebsd-i386),) +ifneq ($(filter $(DEB_HOST_ARCH), hurd-i386 kfreebsd-amd64 kfreebsd-i386),) TESTSUITE_FAIL_CMD=exit 0 endif From 737fe1a53e67a6510264501ff9d039736387f6e8 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 12 Apr 2016 14:44:10 +0200 Subject: [PATCH 16/31] Use POSIX semaphores on hurd --- debian/changelog | 3 ++- debian/patches/hurd-sem_init | 33 +++++++++++++++++++++++++++++++++ debian/patches/series | 1 + debian/rules | 1 - 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 debian/patches/hurd-sem_init diff --git a/debian/changelog b/debian/changelog index 9553267..2aa3957 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,9 @@ postgresql-9.5 (9.5.2-2) UNRELEASED; urgency=medium * rules: Use host architecture for all architecture checks. + * Use POSIX semaphores on hurd. Closes: #820743. - -- Christoph Berg Tue, 12 Apr 2016 11:59:21 +0200 + -- Christoph Berg Tue, 12 Apr 2016 14:45:03 +0200 postgresql-9.5 (9.5.2-1) unstable; urgency=medium diff --git a/debian/patches/hurd-sem_init b/debian/patches/hurd-sem_init new file mode 100644 index 0000000..4a4710f --- /dev/null +++ b/debian/patches/hurd-sem_init @@ -0,0 +1,33 @@ +hurd needs sysv semaphores, and -pthread to find sem_init + +FATAL: could not create semaphores: Function not implemented +DETAIL: Failed system call was semget(1, 17, 03600). + +undefined reference to symbol 'sem_init@@GLIBC_2.12' + +--- a/src/backend/Makefile ++++ b/src/backend/Makefile +@@ -105,6 +105,10 @@ endif + + endif # aix + ++ifeq ($(shell dpkg-architecture -qDEB_HOST_ARCH_OS), hurd) ++LIBS += -pthread ++endif # hurd ++ + # Update the commonly used headers before building the subdirectories + $(SUBDIRS:%=%-recursive): $(top_builddir)/src/include/parser/gram.h $(top_builddir)/src/include/catalog/schemapg.h $(top_builddir)/src/include/utils/fmgroids.h $(top_builddir)/src/include/utils/errcodes.h $(top_builddir)/src/include/utils/probes.h + +--- a/src/template/linux ++++ b/src/template/linux +@@ -28,3 +28,10 @@ if test "$SUN_STUDIO_CC" = "yes" ; then + ;; + esac + fi ++ ++# force use of POSIX instead of SysV semaphores on hurd-i386 ++case $(dpkg-architecture -qDEB_HOST_ARCH) in ++ hurd*) ++ USE_UNNAMED_POSIX_SEMAPHORES=1 ++ ;; ++esac diff --git a/debian/patches/series b/debian/patches/series index 562acd0..dfa51ac 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -6,3 +6,4 @@ 54-debian-alternatives-for-external-tools.patch 64-pg_upgrade-sockdir 70-history +hurd-sem_init diff --git a/debian/rules b/debian/rules index 28e1b0d..a1a2067 100755 --- a/debian/rules +++ b/debian/rules @@ -86,7 +86,6 @@ endif # build should fail on test suite failures on all arches TESTSUITE_FAIL_CMD=exit 1 -# hurd doesn't implement semaphores; succeed anyway so they at least have libpq5 # plperl currently fails on kfreebsd-* ifneq ($(filter $(DEB_HOST_ARCH), hurd-i386 kfreebsd-amd64 kfreebsd-i386),) TESTSUITE_FAIL_CMD=exit 0 From 3b72f53891bbd33d2d61e58fe8f4f2e436af6da3 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 12 Apr 2016 16:21:08 +0200 Subject: [PATCH 17/31] Update comments on semaphore (non-)support on hurd --- debian/changelog | 3 ++- debian/patches/hurd-sem_init | 5 ++++- debian/rules | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 2aa3957..10943d7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,7 +1,8 @@ postgresql-9.5 (9.5.2-2) UNRELEASED; urgency=medium * rules: Use host architecture for all architecture checks. - * Use POSIX semaphores on hurd. Closes: #820743. + * Use POSIX semaphores on hurd. They don't work yet, but have better chances + of eventually getting implemented. Closes: #820743. -- Christoph Berg Tue, 12 Apr 2016 14:45:03 +0200 diff --git a/debian/patches/hurd-sem_init b/debian/patches/hurd-sem_init index 4a4710f..1a2751a 100644 --- a/debian/patches/hurd-sem_init +++ b/debian/patches/hurd-sem_init @@ -1,4 +1,7 @@ -hurd needs sysv semaphores, and -pthread to find sem_init +hurd doesn't support sysv semaphores (semget), and needs -pthread to find +sem_init. POSIX semaphores shared between processes (sem_init(pshared = 1)) +aren't supported yet either, but have the best chance to get implemented, so be +prepared. FATAL: could not create semaphores: Function not implemented DETAIL: Failed system call was semget(1, 17, 03600). diff --git a/debian/rules b/debian/rules index a1a2067..bb13729 100755 --- a/debian/rules +++ b/debian/rules @@ -86,6 +86,7 @@ endif # build should fail on test suite failures on all arches TESTSUITE_FAIL_CMD=exit 1 +# hurd doesn't implement semaphores shared between processes yet; succeed anyway so they at least have libpq5 # plperl currently fails on kfreebsd-* ifneq ($(filter $(DEB_HOST_ARCH), hurd-i386 kfreebsd-amd64 kfreebsd-i386),) TESTSUITE_FAIL_CMD=exit 0 From 16768ca9d6bca40b0b8bf299f2b08aee8af99607 Mon Sep 17 00:00:00 2001 From: Christoph Berg Date: Tue, 3 May 2016 12:15:49 +0200 Subject: [PATCH 18/31] PostgreSQL 9.5 needs the PG_OOM_ADJUST_FILE variable from pg_ctlcluster Introduced in postgresql-common 158 --- debian/changelog | 4 +++- debian/control | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 10943d7..c3e3c7d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,8 +3,10 @@ postgresql-9.5 (9.5.2-2) UNRELEASED; urgency=medium * rules: Use host architecture for all architecture checks. * Use POSIX semaphores on hurd. They don't work yet, but have better chances of eventually getting implemented. Closes: #820743. + * Bump minimum postgresql-common version for postgresql-9.5 to 158 due to + PG_OOM_ADJUST_FILE. - -- Christoph Berg Tue, 12 Apr 2016 14:45:03 +0200 + -- Christoph Berg Tue, 03 May 2016 12:17:09 +0200 postgresql-9.5 (9.5.2-1) unstable; urgency=medium diff --git a/debian/control b/debian/control index 7a27474..6e4227e 100644 --- a/debian/control +++ b/debian/control @@ -132,7 +132,7 @@ Architecture: any Depends: ${misc:Depends}, ${shlibs:Depends}, postgresql-client-9.5, - postgresql-common (>= 142~), + postgresql-common (>= 158~), tzdata, ssl-cert, locales From 79affa6be9758b0a7af35d7cca3cd38002712e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Fri, 21 Oct 2016 12:39:34 +0200 Subject: [PATCH 19/31] Revert "PostgreSQL 9.5 needs the PG_OOM_ADJUST_FILE variable from pg_ctlcluster" This reverts commit 16768ca9d6bca40b0b8bf299f2b08aee8af99607. --- debian/changelog | 4 +--- debian/control | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index a0472ba..c63a090 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,10 +3,8 @@ postgresql-9.5 (9.5.2-2) UNRELEASED; urgency=medium * rules: Use host architecture for all architecture checks. * Use POSIX semaphores on hurd. They don't work yet, but have better chances of eventually getting implemented. Closes: #820743. - * Bump minimum postgresql-common version for postgresql-9.5 to 158 due to - PG_OOM_ADJUST_FILE. - -- Christoph Berg Tue, 03 May 2016 12:17:09 +0200 + -- Christoph Berg Tue, 12 Apr 2016 14:45:03 +0200 postgresql-9.5 (9.5.2-1) unstable; urgency=medium diff --git a/debian/control b/debian/control index 6e4227e..7a27474 100644 --- a/debian/control +++ b/debian/control @@ -132,7 +132,7 @@ Architecture: any Depends: ${misc:Depends}, ${shlibs:Depends}, postgresql-client-9.5, - postgresql-common (>= 158~), + postgresql-common (>= 142~), tzdata, ssl-cert, locales From a751172855659f572b4f9b4db9a9c392cacad956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Fri, 21 Oct 2016 12:44:57 +0200 Subject: [PATCH 20/31] Release 9.5.2-2ubuntu1 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index c63a090..97aa7ef 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +postgresql-9.5 (9.5.2-2ubuntu1) precise; urgency=low + + * Release 9.5.2 for CartoDB + + -- keybase.io/zenitram Thu, 14 Apr 2016 10:41:31 +0000 + postgresql-9.5 (9.5.2-2) UNRELEASED; urgency=medium * rules: Use host architecture for all architecture checks. From cfdb96446de4c4a34ba0fbd3a4942878493e66e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Fri, 21 Oct 2016 12:56:26 +0200 Subject: [PATCH 21/31] Patch to add interrupt handler to PL/Python functions --- debian/patches/92-plpython-interrupt.patch | 66 ++++++++++++++++++++++ debian/patches/series | 1 + 2 files changed, 67 insertions(+) create mode 100644 debian/patches/92-plpython-interrupt.patch diff --git a/debian/patches/92-plpython-interrupt.patch b/debian/patches/92-plpython-interrupt.patch new file mode 100644 index 0000000..255e22b --- /dev/null +++ b/debian/patches/92-plpython-interrupt.patch @@ -0,0 +1,66 @@ +From c7f7c818d90b8f94103b218732ba749db6812ea9 Mon Sep 17 00:00:00 2001 +From: Mario de Frutos +Date: Fri, 14 Oct 2016 18:44:31 +0200 +Subject: [PATCH] Handler to manage interrupts for python code inside PLPython + +--- + src/pl/plpython/plpy_main.c | 35 +++++++++++++++++++++++++++++++++++ + 1 file changed, 35 insertions(+) + +diff --git a/src/pl/plpython/plpy_main.c b/src/pl/plpython/plpy_main.c +index 5dd86c6..b2632cd 100644 +--- a/src/pl/plpython/plpy_main.c ++++ b/src/pl/plpython/plpy_main.c +@@ -74,6 +74,10 @@ PyObject *PLy_interp_globals = NULL; + /* this doesn't need to be global; use PLy_current_execution_context() */ + static PLyExecutionContext *PLy_execution_contexts = NULL; + ++/* postgres backend handler for interruption */ ++static pqsigfunc coreIntHandler = 0; ++static void PLy_handle_interrupt(int sig); ++ + + void + _PG_init(void) +@@ -81,6 +85,9 @@ _PG_init(void) + int **bitmask_ptr; + const int **version_ptr; + ++ // Catch and process signals ++ coreIntHandler = pqsignal(SIGINT, PLy_handle_interrupt); ++ + /* + * Set up a shared bitmask variable telling which Python version(s) are + * loaded into this process's address space. If there's more than one, we +@@ -454,3 +461,31 @@ PLy_pop_execution_context(void) + MemoryContextDelete(context->scratch_ctx); + PLy_free(context); + } ++ ++void _PG_fini(void); ++void ++_PG_fini(void) ++{ ++ if (coreIntHandler) { ++ pqsignal(SIGINT, coreIntHandler); ++ } ++} ++ ++static int ++PLy_python_interruption_handler() ++{ ++ PyErr_SetString(PyExc_RuntimeError, "Execution of function interrupted by signal"); ++ return 0; ++} ++ ++static void ++PLy_handle_interrupt(int sig) ++{ ++ // custom interruption ++ Py_AddPendingCall(PLy_python_interruption_handler, NULL); ++ ++ if (coreIntHandler) { ++ (*coreIntHandler)(sig); ++ } ++} ++ diff --git a/debian/patches/series b/debian/patches/series index 0491b24..9264dfb 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -8,4 +8,5 @@ 70-history 90-profile_fdw.patch 91-extension_shipping.patch +92-plpython-interrupt.patch hurd-sem_init From ea71b65c19636d6e7310bdf1b268d1aee32a5847 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Mart=C3=ADnez?= Date: Fri, 21 Oct 2016 12:56:57 +0200 Subject: [PATCH 22/31] Release 9.5.2-3cdb1 --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index 97aa7ef..e1d5205 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +postgresql-9.5 (9.5.2-3cdb1) precise; urgency=low + + * Release 9.5.2-3cdb1 + * Patch to allow interruptible PLPython functions + + -- keybase.io/zenitram Fri, 21 Oct 2016 10:52:39 +0000 + postgresql-9.5 (9.5.2-2ubuntu1) precise; urgency=low * Release 9.5.2 for CartoDB From 496791c2215f26b8a4a5c9478c8eae85e819ee20 Mon Sep 17 00:00:00 2001 From: Antonio Zamorano Date: Thu, 24 Nov 2016 09:30:19 +0100 Subject: [PATCH 23/31] Release 9.5.2-3cdb2 --- debian/changelog | 7 ++ ...-plpython-segfault-excetion-handling.patch | 81 +++++++++++++++++++ debian/patches/series | 1 + 3 files changed, 89 insertions(+) create mode 100644 debian/patches/93-plpython-segfault-excetion-handling.patch diff --git a/debian/changelog b/debian/changelog index e1d5205..f45094e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +postgresql-9.5 (9.5.2-3cdb2) precise; urgency=low + + * Release 9.5.2-3cdb2 + * Patch to fix segfault in plpython's exception handling + + -- Antonio Zamorano Wed, 23 Nov 2016 15:39:43 +0100 + postgresql-9.5 (9.5.2-3cdb1) precise; urgency=low * Release 9.5.2-3cdb1 diff --git a/debian/patches/93-plpython-segfault-excetion-handling.patch b/debian/patches/93-plpython-segfault-excetion-handling.patch new file mode 100644 index 0000000..84d88e9 --- /dev/null +++ b/debian/patches/93-plpython-segfault-excetion-handling.patch @@ -0,0 +1,81 @@ +From e95649d8d173f483d67d9338a9089947a7667e5f Mon Sep 17 00:00:00 2001 +From: Rafa de la Torre +Date: Wed, 23 Nov 2016 10:50:40 +0100 +Subject: [PATCH] Fix segfault in plpython's exception handling + +When a subtransaction is aborted in plpython because of an SPI +exception, it tries to find a matching python exception in a hash +`PLy_spi_exceptions` and to make python vm raise it. + +That hash is generated during module initialization, but the exception +objects are not marked to prevent the garbage collector from collecting +them, which can lead to a segmentation fault when processing any SPI +exception. + +PoC to reproduce the issue: + +```sql +CREATE OR REPLACE FUNCTION server_crashes() +RETURNS VOID +AS $$ + import gc + gc.collect() + plan = plpy.prepare('SELECT raises_an_spi_exception();', []) + plpy.execute(plan) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION raises_an_spi_exception() +RETURNS VOID +AS $$ +DECLARE + sql TEXT; +BEGIN + sql = format('%I', NULL); -- NullValueNotAllowed +END +$$ LANGUAGE plpgsql; + +SELECT server_crashes(); -- segfault here +``` + +Stacktrace of the problem (using PostgreSQL `REL9_5_STABLE` and python +`2.7.3-0ubuntu3.8` on a Ubuntu 12.04): + +``` + Program received signal SIGSEGV, Segmentation fault. + 0x00007f3155c7670b in PyObject_Call (func=0x7f31b7db2a30, arg=0x7f31b87d17d0, kw=0x0) at ../Objects/abstract.c:2525 + 2525 ../Objects/abstract.c: No such file or directory. + (gdb) bt + #0 0x00007f3155c7670b in PyObject_Call (func=0x7f31b7db2a30, arg=0x7f31b87d17d0, kw=0x0) at ../Objects/abstract.c:2525 + #1 0x00007f3155d81ab1 in PyEval_CallObjectWithKeywords (func=0x7f31b7db2a30, arg=0x7f31b87d17d0, kw=0x0) at ../Python/ceval.c:3890 + #2 0x00007f3155c766ed in PyObject_CallObject (o=0x7f31b7db2a30, a=0x7f31b87d17d0) at ../Objects/abstract.c:2517 + #3 0x00007f31561e112b in PLy_spi_exception_set (edata=0x7f31b8743d78, excclass=0x7f31b7db2a30) at plpy_spi.c:547 + #4 PLy_spi_subtransaction_abort (oldcontext=, oldowner=) at plpy_spi.c:527 + #5 0x00007f31561e2185 in PLy_spi_execute_plan (ob=0x7f31b87d0cd8, list=0x7f31b7c530d8, limit=0) at plpy_spi.c:307 + #6 0x00007f31561e22d4 in PLy_spi_execute (self=, args=0x7f31b87a6d08) at plpy_spi.c:180 + #7 0x00007f3155cda4d6 in PyCFunction_Call (func=0x7f31b7d29600, arg=0x7f31b87a6d08, kw=0x0) at ../Objects/methodobject.c:81 + #8 0x00007f3155d82383 in call_function (pp_stack=0x7fff9207e710, oparg=2) at ../Python/ceval.c:4021 + #9 0x00007f3155d7cda4 in PyEval_EvalFrameEx (f=0x7f31b8805be0, throwflag=0) at ../Python/ceval.c:2666 + #10 0x00007f3155d82898 in fast_function (func=0x7f31b88b5ed0, pp_stack=0x7fff9207ea70, n=0, na=0, nk=0) at ../Python/ceval.c:4107 + #11 0x00007f3155d82584 in call_function (pp_stack=0x7fff9207ea70, oparg=0) at ../Python/ceval.c:4042 + #12 0x00007f3155d7cda4 in PyEval_EvalFrameEx (f=0x7f31b8805a00, throwflag=0) at ../Python/ceval.c:2666 + #13 0x00007f3155d7f8a9 in PyEval_EvalCodeEx (co=0x7f31b88aa460, globals=0x7f31b8727ea0, locals=0x7f31b8727ea0, args=0x0, argcount=0, kws=0x0, kwcount=0, defs=0x0, defcount=0, closure=0x0) at ../Python/ceval.c:3253 + #14 0x00007f3155d74ff4 in PyEval_EvalCode (co=0x7f31b88aa460, globals=0x7f31b8727ea0, locals=0x7f31b8727ea0) at ../Python/ceval.c:667 + #15 0x00007f31561dc476 in PLy_procedure_call (kargs=kargs@entry=0x7f31561e5690 "args", vargs=, proc=0x7f31b873b2d0, proc=0x7f31b873b2d0) at plpy_exec.c:801 + #16 0x00007f31561dd9c6 in PLy_exec_function (fcinfo=fcinfo@entry=0x7f31b7c1f870, proc=0x7f31b873b2d0) at plpy_exec.c:61#17 0x00007f31561de9f9 in plpython_call_handler (fcinfo=0x7f31b7c1f870) at plpy_main.c:291 +``` +--- + src/pl/plpython/plpy_plpymodule.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/pl/plpython/plpy_plpymodule.c b/src/pl/plpython/plpy_plpymodule.c +index a44b7fb..3334739 100644 +--- a/src/pl/plpython/plpy_plpymodule.c ++++ b/src/pl/plpython/plpy_plpymodule.c +@@ -258,6 +258,7 @@ PLy_generate_spi_exceptions(PyObject *mod, PyObject *base) + PyDict_SetItemString(dict, "sqlstate", sqlstate); + Py_DECREF(sqlstate); + exc = PyErr_NewException(exception_map[i].name, base, dict); ++ Py_INCREF(exc); + PyModule_AddObject(mod, exception_map[i].classname, exc); + entry = hash_search(PLy_spi_exceptions, &exception_map[i].sqlstate, + HASH_ENTER, &found); diff --git a/debian/patches/series b/debian/patches/series index 9264dfb..a70895c 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -9,4 +9,5 @@ 90-profile_fdw.patch 91-extension_shipping.patch 92-plpython-interrupt.patch +93-plpython-segfault-excetion-handling.patch hurd-sem_init From 1af0aec5654b712cf3a5d20b0d1ee4303abcce02 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 21 Apr 2017 18:54:51 +0200 Subject: [PATCH 24/31] Fix problem with INT signals outside of plpython execution INT signals received when python code is not being executed affected code executed later. --- debian/patches/92-plpython-interrupt.patch | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/debian/patches/92-plpython-interrupt.patch b/debian/patches/92-plpython-interrupt.patch index 255e22b..5029e39 100644 --- a/debian/patches/92-plpython-interrupt.patch +++ b/debian/patches/92-plpython-interrupt.patch @@ -57,7 +57,11 @@ index 5dd86c6..b2632cd 100644 +PLy_handle_interrupt(int sig) +{ + // custom interruption -+ Py_AddPendingCall(PLy_python_interruption_handler, NULL); ++ if (PLy_execution_contexts != NULL) { ++ // Insert python interruption only if a python function is being executed ++ // otherwise, future python execution would be affected ++ Py_AddPendingCall(PLy_python_interruption_handler, NULL); ++ } + + if (coreIntHandler) { + (*coreIntHandler)(sig); From 85d9a312b39e9d0d12358aecf17060a1a0cf2643 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 25 Apr 2017 11:57:57 +0200 Subject: [PATCH 25/31] More strict plpython clean up The previous signal handler should always be restore upon module clean up. Note that a null pointer may be a valid handler specifier (SIG_DFL actually). We rely on _PG_fini calls being always paired with _PG_init. Note also that as of PostgreSQL 9.6 _PG_init isn't actually called ever (modules are never unloaded). --- debian/patches/92-plpython-interrupt.patch | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/debian/patches/92-plpython-interrupt.patch b/debian/patches/92-plpython-interrupt.patch index 5029e39..f84a618 100644 --- a/debian/patches/92-plpython-interrupt.patch +++ b/debian/patches/92-plpython-interrupt.patch @@ -41,9 +41,7 @@ index 5dd86c6..b2632cd 100644 +void +_PG_fini(void) +{ -+ if (coreIntHandler) { -+ pqsignal(SIGINT, coreIntHandler); -+ } ++ pqsignal(SIGINT, coreIntHandler); +} + +static int From 3644014d29826fead496ad896291ee591c4aa8e7 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 3 May 2017 10:11:07 +0200 Subject: [PATCH 26/31] Avoid potential problems with SIG_DFL, SIG_IGN --- debian/patches/92-plpython-interrupt.patch | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/patches/92-plpython-interrupt.patch b/debian/patches/92-plpython-interrupt.patch index f84a618..3405289 100644 --- a/debian/patches/92-plpython-interrupt.patch +++ b/debian/patches/92-plpython-interrupt.patch @@ -61,7 +61,8 @@ index 5dd86c6..b2632cd 100644 + Py_AddPendingCall(PLy_python_interruption_handler, NULL); + } + -+ if (coreIntHandler) { ++ if (coreIntHandler != SIG_DFL && coreIntHandler != SIG_IGN) { ++ // Note that we're handling SIG_DFL (incorrectly) as SIG_IGN + (*coreIntHandler)(sig); + } +} From d39984ddafec622772c2238e7f955c7e35ab72ed Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 3 May 2017 16:18:52 +0200 Subject: [PATCH 27/31] Fix pending call return value. Returning 0 from the pending call function has the effect of not raising errors immediately. We must return -1 to make sure any errors are trapped in the current context. --- debian/patches/92-plpython-interrupt.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/patches/92-plpython-interrupt.patch b/debian/patches/92-plpython-interrupt.patch index 3405289..d492ff5 100644 --- a/debian/patches/92-plpython-interrupt.patch +++ b/debian/patches/92-plpython-interrupt.patch @@ -48,7 +48,7 @@ index 5dd86c6..b2632cd 100644 +PLy_python_interruption_handler() +{ + PyErr_SetString(PyExc_RuntimeError, "Execution of function interrupted by signal"); -+ return 0; ++ return -1; +} + +static void From ac37ab134d1d9d2f97037d9fa2259372e1f8d683 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 3 May 2017 17:08:08 +0200 Subject: [PATCH 28/31] Prevent out-of-context Python interruptions Make sure Python exceptions occur in the context where the SIGINT occured --- debian/patches/92-plpython-interrupt.patch | 64 +++++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/debian/patches/92-plpython-interrupt.patch b/debian/patches/92-plpython-interrupt.patch index d492ff5..f1fb415 100644 --- a/debian/patches/92-plpython-interrupt.patch +++ b/debian/patches/92-plpython-interrupt.patch @@ -14,25 +14,52 @@ index 5dd86c6..b2632cd 100644 @@ -74,6 +74,10 @@ PyObject *PLy_interp_globals = NULL; /* this doesn't need to be global; use PLy_current_execution_context() */ static PLyExecutionContext *PLy_execution_contexts = NULL; - + +/* postgres backend handler for interruption */ +static pqsigfunc coreIntHandler = 0; +static void PLy_handle_interrupt(int sig); -+ - + void _PG_init(void) -@@ -81,6 +85,9 @@ _PG_init(void) +@@ -81,6 +84,9 @@ _PG_init(void) int **bitmask_ptr; const int **version_ptr; - -+ // Catch and process signals -+ coreIntHandler = pqsignal(SIGINT, PLy_handle_interrupt); + ++ // Catch and process signals ++ coreIntHandler = pqsignal(SIGINT, PLy_handle_interrupt); + /* * Set up a shared bitmask variable telling which Python version(s) are * loaded into this process's address space. If there's more than one, we -@@ -454,3 +461,31 @@ PLy_pop_execution_context(void) +@@ -425,6 +431,9 @@ PLy_current_execution_context(void) + return PLy_execution_contexts; + } + ++static int PLy_execution_context_level = 0; ++static int PLy_pending_interrupt = 0; ++ + static PLyExecutionContext * + PLy_push_execution_context(void) + { +@@ -438,6 +447,7 @@ PLy_push_execution_context(void) + ALLOCSET_DEFAULT_MAXSIZE); + context->next = PLy_execution_contexts; + PLy_execution_contexts = context; ++ ++PLy_execution_context_level; + return context; + } + +@@ -449,8 +459,48 @@ PLy_pop_execution_context(void) + if (context == NULL) + elog(ERROR, "no Python function is currently executing"); + ++ if (--PLy_execution_context_level == 0) { ++ // Clear pending interrupts when top level context exits ++ PLy_pending_interrupt = 0; ++ } ++ + PLy_execution_contexts = context->next; + MemoryContextDelete(context->scratch_ctx); PLy_free(context); } @@ -41,12 +68,18 @@ index 5dd86c6..b2632cd 100644 +void +_PG_fini(void) +{ -+ pqsignal(SIGINT, coreIntHandler); ++ // Restore previous SIGINT handler ++ pqsignal(SIGINT, coreIntHandler); +} + +static int +PLy_python_interruption_handler() +{ ++ if (!PLy_pending_interrupt) { ++ return 0; ++ } ++ ++ PLy_pending_interrupt = 0; + PyErr_SetString(PyExc_RuntimeError, "Execution of function interrupted by signal"); + return -1; +} @@ -54,16 +87,17 @@ index 5dd86c6..b2632cd 100644 +static void +PLy_handle_interrupt(int sig) +{ -+ // custom interruption -+ if (PLy_execution_contexts != NULL) { ++ if (PLy_execution_context_level > 0) { + // Insert python interruption only if a python function is being executed + // otherwise, future python execution would be affected -+ Py_AddPendingCall(PLy_python_interruption_handler, NULL); ++ if (!PLy_pending_interrupt) { ++ PLy_pending_interrupt = 1; ++ Py_AddPendingCall(PLy_python_interruption_handler, PLy_execution_contexts); ++ } + } -+ ++ // Fallback to prior handlers + if (coreIntHandler != SIG_DFL && coreIntHandler != SIG_IGN) { -+ // Note that we're handling SIG_DFL (incorrectly) as SIG_IGN ++ Note that we're handling SIG_DFL (incorrectly) as SIG_IGN + (*coreIntHandler)(sig); + } +} -+ From 5562521671f4b9c9cdb1f804e0c724cb986ea85c Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 5 May 2017 12:09:36 +0200 Subject: [PATCH 29/31] Simplify plpython context tracking Remove unnecessary remnants from previous tests: * No need to keep track of plpython context nesting level * No need to pass arguments to the python pending call --- debian/patches/92-plpython-interrupt.patch | 35 +++++++--------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/debian/patches/92-plpython-interrupt.patch b/debian/patches/92-plpython-interrupt.patch index f1fb415..a9a5380 100644 --- a/debian/patches/92-plpython-interrupt.patch +++ b/debian/patches/92-plpython-interrupt.patch @@ -35,31 +35,20 @@ index 5dd86c6..b2632cd 100644 return PLy_execution_contexts; } -+static int PLy_execution_context_level = 0; ++/* Indicate tha a python interruption is pending */ +static int PLy_pending_interrupt = 0; + static PLyExecutionContext * PLy_push_execution_context(void) { -@@ -438,6 +447,7 @@ PLy_push_execution_context(void) - ALLOCSET_DEFAULT_MAXSIZE); - context->next = PLy_execution_contexts; - PLy_execution_contexts = context; -+ ++PLy_execution_context_level; - return context; - } - -@@ -449,8 +459,48 @@ PLy_pop_execution_context(void) - if (context == NULL) - elog(ERROR, "no Python function is currently executing"); +@@ -451,6 +460,47 @@ PLy_pop_execution_context(void) + PLy_execution_contexts = context->next; -+ if (--PLy_execution_context_level == 0) { ++ if (PLy_execution_contexts == NULL) { + // Clear pending interrupts when top level context exits + PLy_pending_interrupt = 0; + } + - PLy_execution_contexts = context->next; - MemoryContextDelete(context->scratch_ctx); PLy_free(context); } @@ -87,17 +76,15 @@ index 5dd86c6..b2632cd 100644 +static void +PLy_handle_interrupt(int sig) +{ -+ if (PLy_execution_context_level > 0) { -+ // Insert python interruption only if a python function is being executed -+ // otherwise, future python execution would be affected -+ if (!PLy_pending_interrupt) { -+ PLy_pending_interrupt = 1; -+ Py_AddPendingCall(PLy_python_interruption_handler, PLy_execution_contexts); -+ } ++ if (PLy_execution_contexts != NULL && !PLy_pending_interrupt) { ++ PLy_pending_interrupt = 1; ++ Py_AddPendingCall(PLy_python_interruption_handler, NULL); + } -+ // Fallback to prior handlers ++ // Fallback to execute prior handlers + if (coreIntHandler != SIG_DFL && coreIntHandler != SIG_IGN) { -+ Note that we're handling SIG_DFL (incorrectly) as SIG_IGN ++ // There's a catch here: if the prior handler was SIG_DFL we have no easy way ++ // of invoking it here; ++ // As that's an unlikely situation we'll just treat SIG_DFL as SIG_IGN. + (*coreIntHandler)(sig); + } +} From 34d59936538cd766135fc75ff8ef27073f4a6a71 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 23 May 2017 18:24:55 +0200 Subject: [PATCH 30/31] New patch based on jgoizueta's work See https://github.com/CartoDB/postgres/pull/5 and more specifically the commit b319cee --- debian/patches/92-plpython-interrupt.patch | 35 +++++++++++----------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/debian/patches/92-plpython-interrupt.patch b/debian/patches/92-plpython-interrupt.patch index a9a5380..b07cf50 100644 --- a/debian/patches/92-plpython-interrupt.patch +++ b/debian/patches/92-plpython-interrupt.patch @@ -1,31 +1,30 @@ -From c7f7c818d90b8f94103b218732ba749db6812ea9 Mon Sep 17 00:00:00 2001 -From: Mario de Frutos -Date: Fri, 14 Oct 2016 18:44:31 +0200 -Subject: [PATCH] Handler to manage interrupts for python code inside PLPython +commit b319ceef00d3a9cfd5b7a4bd0bc9a892ca18149b +Author: Javier Goizueta +Date: Fri May 5 12:27:15 2017 +0200 ---- - src/pl/plpython/plpy_main.c | 35 +++++++++++++++++++++++++++++++++++ - 1 file changed, 35 insertions(+) + Make execution of plpython interruptible + + The plpy module hooks into the signal handling mechanism to insert python exceptions when a SIGINT occurs. diff --git a/src/pl/plpython/plpy_main.c b/src/pl/plpython/plpy_main.c -index 5dd86c6..b2632cd 100644 +index 5dd86c6..9f53453 100644 --- a/src/pl/plpython/plpy_main.c +++ b/src/pl/plpython/plpy_main.c -@@ -74,6 +74,10 @@ PyObject *PLy_interp_globals = NULL; +@@ -74,6 +74,9 @@ PyObject *PLy_interp_globals = NULL; /* this doesn't need to be global; use PLy_current_execution_context() */ static PLyExecutionContext *PLy_execution_contexts = NULL; - + +/* postgres backend handler for interruption */ +static pqsigfunc coreIntHandler = 0; +static void PLy_handle_interrupt(int sig); - + void _PG_init(void) @@ -81,6 +84,9 @@ _PG_init(void) int **bitmask_ptr; const int **version_ptr; - -+ // Catch and process signals + ++ /* Catch and process SIGINT signals */ + coreIntHandler = pqsignal(SIGINT, PLy_handle_interrupt); + /* @@ -34,16 +33,17 @@ index 5dd86c6..b2632cd 100644 @@ -425,6 +431,9 @@ PLy_current_execution_context(void) return PLy_execution_contexts; } - + +/* Indicate tha a python interruption is pending */ +static int PLy_pending_interrupt = 0; + static PLyExecutionContext * PLy_push_execution_context(void) { -@@ -451,6 +460,47 @@ PLy_pop_execution_context(void) - PLy_execution_contexts = context->next; - +@@ -451,6 +460,46 @@ PLy_pop_execution_context(void) + + PLy_execution_contexts = context->next; + + if (PLy_execution_contexts == NULL) { + // Clear pending interrupts when top level context exits + PLy_pending_interrupt = 0; @@ -53,7 +53,6 @@ index 5dd86c6..b2632cd 100644 PLy_free(context); } + -+void _PG_fini(void); +void +_PG_fini(void) +{ From 12b89e033b494b62ccd4bb7f774d0032fe71f910 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 23 May 2017 18:51:13 +0200 Subject: [PATCH 31/31] Release 9.5.2-3cdb3 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index f45094e..d6e15ff 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +postgresql-9.5 (9.5.2-3cdb3) precise; urgency=medium + + * Improved patch for interruptible PLPython functions + + -- Rafa de la Torre Tue, 23 May 2017 18:48:02 +0200 + postgresql-9.5 (9.5.2-3cdb2) precise; urgency=low * Release 9.5.2-3cdb2