From f631496d0b4064c9a8d38c40187842f326d64cbc Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Mon, 16 Jun 2025 17:59:40 +0200 Subject: [PATCH 01/21] PG-1663 Make sure indexes on paritioned tables are encrypted Since we only looked at the parent table and not on the whole tree when setting the status of the encrypted indexes we could easily accidentally create a plain text index on an encrypted table. This patch also makes sure to disallow adding indexes to an inheritance tree where the tables are a mix of encrypted and unecrypted tables. --- contrib/pg_tde/expected/partition_table.out | 46 +++++++++++++++++++++ contrib/pg_tde/sql/partition_table.sql | 29 +++++++++++++ contrib/pg_tde/src/pg_tde_event_capture.c | 21 +++++----- 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/contrib/pg_tde/expected/partition_table.out b/contrib/pg_tde/expected/partition_table.out index e58c181d5bca2..13553c5e0134a 100644 --- a/contrib/pg_tde/expected/partition_table.out +++ b/contrib/pg_tde/expected/partition_table.out @@ -161,4 +161,50 @@ SELECT pg_tde_is_encrypted('partition_child_tde_heap'); DROP TABLE partition_parent; RESET pg_tde.enforce_encryption; +-- Partitioned indexes should be encrypted +CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); +CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING tde_heap; +CREATE INDEX ON partition_parent (a); +SELECT pg_tde_is_encrypted('partition_parent_a_idx'); -- Also check that the parent index is NULL + pg_tde_is_encrypted +--------------------- + +(1 row) + +SELECT pg_tde_is_encrypted('partition_child_a_idx'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +DROP TABLE partition_parent; +-- Partitioned indexes should be not encrypted with heap +CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); +CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING heap; +CREATE INDEX ON partition_parent (a); +SELECT pg_tde_is_encrypted('partition_child_a_idx'); + pg_tde_is_encrypted +--------------------- + f +(1 row) + +DROP TABLE partition_parent; +-- We refuse to create an index when the inheritance heirarchy has mixed statuses +CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); +CREATE TABLE partition_child_heap PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING heap; +CREATE TABLE partition_child_tde_heap PARTITION OF partition_parent FOR VALUES FROM (10) TO (19) USING tde_heap; +CREATE INDEX ON partition_parent (a); +ERROR: Recursive CREATE INDEX on a mix of encrypted and unencrypted relations is not supported +DROP TABLE partition_parent; +-- Index should also be encrypted for new partitionins +CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); +CREATE INDEX ON partition_parent (a); +CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (10) TO (19) USING tde_heap; +SELECT pg_tde_is_encrypted('partition_child_a_idx'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +DROP TABLE partition_parent; DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/partition_table.sql b/contrib/pg_tde/sql/partition_table.sql index 0885e55930c10..71ce2a1a7b28e 100644 --- a/contrib/pg_tde/sql/partition_table.sql +++ b/contrib/pg_tde/sql/partition_table.sql @@ -80,4 +80,33 @@ SELECT pg_tde_is_encrypted('partition_child_tde_heap'); DROP TABLE partition_parent; RESET pg_tde.enforce_encryption; +-- Partitioned indexes should be encrypted +CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); +CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING tde_heap; +CREATE INDEX ON partition_parent (a); +SELECT pg_tde_is_encrypted('partition_parent_a_idx'); -- Also check that the parent index is NULL +SELECT pg_tde_is_encrypted('partition_child_a_idx'); +DROP TABLE partition_parent; + +-- Partitioned indexes should be not encrypted with heap +CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); +CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING heap; +CREATE INDEX ON partition_parent (a); +SELECT pg_tde_is_encrypted('partition_child_a_idx'); +DROP TABLE partition_parent; + +-- We refuse to create an index when the inheritance heirarchy has mixed statuses +CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); +CREATE TABLE partition_child_heap PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING heap; +CREATE TABLE partition_child_tde_heap PARTITION OF partition_parent FOR VALUES FROM (10) TO (19) USING tde_heap; +CREATE INDEX ON partition_parent (a); +DROP TABLE partition_parent; + +-- Index should also be encrypted for new partitionins +CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); +CREATE INDEX ON partition_parent (a); +CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (10) TO (19) USING tde_heap; +SELECT pg_tde_is_encrypted('partition_child_a_idx'); +DROP TABLE partition_parent; + DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/src/pg_tde_event_capture.c b/contrib/pg_tde/src/pg_tde_event_capture.c index 43ee36cbecfd9..900c8e80aa515 100644 --- a/contrib/pg_tde/src/pg_tde_event_capture.c +++ b/contrib/pg_tde/src/pg_tde_event_capture.c @@ -258,21 +258,22 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) if (IsA(parsetree, IndexStmt)) { IndexStmt *stmt = castNode(IndexStmt, parsetree); - Relation rel; TdeDdlEvent event = {.parsetree = parsetree}; + EncryptionMix encmix; + Oid relid = RangeVarGetRelid(stmt->relation, AccessShareLock, false); - rel = table_openrv(stmt->relation, AccessShareLock); + encmix = alter_table_encryption_mix(relid); - if (rel->rd_rel->relam == get_tde_table_am_oid()) - { + if (encmix == ENC_MIX_ENCRYPTED) event.encryptMode = TDE_ENCRYPT_MODE_ENCRYPT; - checkPrincipalKeyConfigured(); - } - else + else if (encmix == ENC_MIX_PLAIN) event.encryptMode = TDE_ENCRYPT_MODE_PLAIN; - - /* Hold on to lock until end of transaction */ - table_close(rel, NoLock); + else if (encmix == ENC_MIX_UNKNOWN) + event.encryptMode = TDE_ENCRYPT_MODE_RETAIN; + else + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("Recursive CREATE INDEX on a mix of encrypted and unencrypted relations is not supported")); push_event_stack(&event); } From 93dcf725d2743a42a14f9c82c12a85d7fcf2f7df Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Wed, 18 Jun 2025 08:39:16 +0100 Subject: [PATCH 02/21] Update postgres and pg_tde version numbers --- ci_scripts/env.sh | 2 +- configure | 2 +- configure.ac | 2 +- contrib/pg_tde/Makefile | 2 +- contrib/pg_tde/expected/version.out | 6 +++--- contrib/pg_tde/meson.build | 2 +- contrib/pg_tde/{pg_tde--1.0-rc.sql => pg_tde--1.0.sql} | 0 contrib/pg_tde/pg_tde.control | 2 +- contrib/pg_tde/src/include/pg_tde.h | 2 +- contrib/pg_tde/t/expected/basic.out | 2 +- meson.build | 2 +- src/test/modules/test_misc/t/008_percona_server_version.pl | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) rename contrib/pg_tde/{pg_tde--1.0-rc.sql => pg_tde--1.0.sql} (100%) diff --git a/ci_scripts/env.sh b/ci_scripts/env.sh index a5bb68ac66560..ec1cff4f8f2a1 100644 --- a/ci_scripts/env.sh +++ b/ci_scripts/env.sh @@ -1,3 +1,3 @@ #!/bin/bash -export PERCONA_SERVER_VERSION=17.5.1 +export PERCONA_SERVER_VERSION=17.5.2 diff --git a/configure b/configure index d255f91601c3b..7960612b26b5c 100755 --- a/configure +++ b/configure @@ -2856,7 +2856,7 @@ cat >>confdefs.h <<_ACEOF _ACEOF -PG_PERCONAVERSION=1 +PG_PERCONAVERSION=2 cat >>confdefs.h <<_ACEOF #define PG_PERCONAVERSION "$PG_PERCONAVERSION" diff --git a/configure.ac b/configure.ac index 669e3bcffc10b..70fa3619d4bad 100644 --- a/configure.ac +++ b/configure.ac @@ -37,7 +37,7 @@ AC_DEFINE_UNQUOTED(PG_MAJORVERSION, "$PG_MAJORVERSION", [PostgreSQL major versio AC_DEFINE_UNQUOTED(PG_MAJORVERSION_NUM, $PG_MAJORVERSION, [PostgreSQL major version number]) AC_DEFINE_UNQUOTED(PG_MINORVERSION_NUM, $PG_MINORVERSION, [PostgreSQL minor version number]) -[PG_PERCONAVERSION=1] +[PG_PERCONAVERSION=2] AC_DEFINE_UNQUOTED(PG_PERCONAVERSION, "$PG_PERCONAVERSION", [PostgreSQL Percona version as a string]) PGAC_ARG_REQ(with, extra-version, [STRING], [append STRING to version], diff --git a/contrib/pg_tde/Makefile b/contrib/pg_tde/Makefile index a8b7458b411f8..dd9f5541c6d19 100644 --- a/contrib/pg_tde/Makefile +++ b/contrib/pg_tde/Makefile @@ -1,7 +1,7 @@ PGFILEDESC = "pg_tde access method" MODULE_big = pg_tde EXTENSION = pg_tde -DATA = pg_tde--1.0-rc.sql +DATA = pg_tde--1.0.sql # Since meson supports skipping test suites this is a make only feature ifndef TDE_MODE diff --git a/contrib/pg_tde/expected/version.out b/contrib/pg_tde/expected/version.out index fddbafca98f12..44ebea8d04d32 100644 --- a/contrib/pg_tde/expected/version.out +++ b/contrib/pg_tde/expected/version.out @@ -1,8 +1,8 @@ CREATE EXTENSION pg_tde; SELECT pg_tde_version(); - pg_tde_version ------------------ - pg_tde 1.0.0-rc + pg_tde_version +---------------- + pg_tde 1.0.0 (1 row) DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index 21f5b13c445a8..7429c08420f93 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -75,7 +75,7 @@ endif install_data( 'pg_tde.control', - 'pg_tde--1.0-rc.sql', + 'pg_tde--1.0.sql', kwargs: contrib_data_args, ) diff --git a/contrib/pg_tde/pg_tde--1.0-rc.sql b/contrib/pg_tde/pg_tde--1.0.sql similarity index 100% rename from contrib/pg_tde/pg_tde--1.0-rc.sql rename to contrib/pg_tde/pg_tde--1.0.sql diff --git a/contrib/pg_tde/pg_tde.control b/contrib/pg_tde/pg_tde.control index 73d135807fbe4..9ea82992d7490 100644 --- a/contrib/pg_tde/pg_tde.control +++ b/contrib/pg_tde/pg_tde.control @@ -1,4 +1,4 @@ comment = 'pg_tde access method' -default_version = '1.0-rc' +default_version = '1.0' module_pathname = '$libdir/pg_tde' relocatable = false diff --git a/contrib/pg_tde/src/include/pg_tde.h b/contrib/pg_tde/src/include/pg_tde.h index 652dc8eea18c6..4b6bb94d6d8f4 100644 --- a/contrib/pg_tde/src/include/pg_tde.h +++ b/contrib/pg_tde/src/include/pg_tde.h @@ -2,7 +2,7 @@ #define PG_TDE_H #define PG_TDE_NAME "pg_tde" -#define PG_TDE_VERSION "1.0.0-rc" +#define PG_TDE_VERSION "1.0.0" #define PG_TDE_VERSION_STRING PG_TDE_NAME " " PG_TDE_VERSION #define PG_TDE_DATA_DIR "pg_tde" diff --git a/contrib/pg_tde/t/expected/basic.out b/contrib/pg_tde/t/expected/basic.out index 587d8a7a31bf9..fdf976fb314c6 100644 --- a/contrib/pg_tde/t/expected/basic.out +++ b/contrib/pg_tde/t/expected/basic.out @@ -20,7 +20,7 @@ CREATE EXTENSION pg_tde; SELECT extname, extversion FROM pg_extension WHERE extname = 'pg_tde'; extname | extversion ---------+------------ - pg_tde | 1.0-rc + pg_tde | 1.0 (1 row) CREATE TABLE test_enc (id SERIAL, k INTEGER, PRIMARY KEY (id)) USING tde_heap; diff --git a/meson.build b/meson.build index 6fbc5163c0171..c054200660486 100644 --- a/meson.build +++ b/meson.build @@ -134,7 +134,7 @@ endif pg_version_major = pg_version_arr[0].to_int() pg_version_minor = pg_version_arr[1].to_int() pg_version_num = (pg_version_major * 10000) + pg_version_minor -pg_percona_ver = '1' +pg_percona_ver = '2' pg_url = 'https://www.postgresql.org/' diff --git a/src/test/modules/test_misc/t/008_percona_server_version.pl b/src/test/modules/test_misc/t/008_percona_server_version.pl index c7723525ebf22..347e173b523ed 100644 --- a/src/test/modules/test_misc/t/008_percona_server_version.pl +++ b/src/test/modules/test_misc/t/008_percona_server_version.pl @@ -17,7 +17,7 @@ # To make this testcase work, PERCONA_SERVER_VERSION variable should be available in environment. # If you are using ci_scripts it is already declated in ci_scripts/env.sh # If you are using command line make for regression then export like: -# export PERCONA_SERVER_VERSION=17.4.1 +# export PERCONA_SERVER_VERSION=17.5.2 if (!defined($ENV{PERCONA_SERVER_VERSION})) { From d48fdea0144ebfd2f74259c766642e6ef97e0038 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Wed, 18 Jun 2025 14:29:49 +0200 Subject: [PATCH 03/21] PG-1662 Handle changing access method of partitioned table correctly Since partitioned tables do not have any sotrage and only control the default access method of their children we should not try to change the encryption status of anything when changing the AM of a partitioned table. --- contrib/pg_tde/expected/partition_table.out | 24 +++++++++++++++++++++ contrib/pg_tde/sql/partition_table.sql | 9 ++++++++ contrib/pg_tde/src/pg_tde_event_capture.c | 13 ++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/contrib/pg_tde/expected/partition_table.out b/contrib/pg_tde/expected/partition_table.out index 13553c5e0134a..55835d3d4f0cd 100644 --- a/contrib/pg_tde/expected/partition_table.out +++ b/contrib/pg_tde/expected/partition_table.out @@ -161,6 +161,30 @@ SELECT pg_tde_is_encrypted('partition_child_tde_heap'); DROP TABLE partition_parent; RESET pg_tde.enforce_encryption; +-- Does not change encryption of child tables when rewriting and changing AM +CREATE TABLE partition_parent (a int, b int) PARTITION BY RANGE (a) USING tde_heap; +CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING tde_heap; +SELECT pg_tde_is_encrypted('partition_child'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +ALTER TABLE partition_parent SET ACCESS METHOD heap, ALTER b TYPE bigint; +SELECT relname, amname FROM pg_class JOIN pg_am ON pg_am.oid = pg_class.relam WHERE relname IN ('partition_parent', 'partition_child') ORDER BY relname; + relname | amname +------------------+---------- + partition_child | tde_heap + partition_parent | heap +(2 rows) + +SELECT pg_tde_is_encrypted('partition_child'); + pg_tde_is_encrypted +--------------------- + t +(1 row) + +DROP TABLE partition_parent; -- Partitioned indexes should be encrypted CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING tde_heap; diff --git a/contrib/pg_tde/sql/partition_table.sql b/contrib/pg_tde/sql/partition_table.sql index 71ce2a1a7b28e..45e125dd0e8d9 100644 --- a/contrib/pg_tde/sql/partition_table.sql +++ b/contrib/pg_tde/sql/partition_table.sql @@ -80,6 +80,15 @@ SELECT pg_tde_is_encrypted('partition_child_tde_heap'); DROP TABLE partition_parent; RESET pg_tde.enforce_encryption; +-- Does not change encryption of child tables when rewriting and changing AM +CREATE TABLE partition_parent (a int, b int) PARTITION BY RANGE (a) USING tde_heap; +CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING tde_heap; +SELECT pg_tde_is_encrypted('partition_child'); +ALTER TABLE partition_parent SET ACCESS METHOD heap, ALTER b TYPE bigint; +SELECT relname, amname FROM pg_class JOIN pg_am ON pg_am.oid = pg_class.relam WHERE relname IN ('partition_parent', 'partition_child') ORDER BY relname; +SELECT pg_tde_is_encrypted('partition_child'); +DROP TABLE partition_parent; + -- Partitioned indexes should be encrypted CREATE TABLE partition_parent (a int) PARTITION BY RANGE (a); CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (9) USING tde_heap; diff --git a/contrib/pg_tde/src/pg_tde_event_capture.c b/contrib/pg_tde/src/pg_tde_event_capture.c index 900c8e80aa515..58bc91d4d8f3f 100644 --- a/contrib/pg_tde/src/pg_tde_event_capture.c +++ b/contrib/pg_tde/src/pg_tde_event_capture.c @@ -357,6 +357,7 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) ListCell *lcmd; TdeDdlEvent event = {.parsetree = parsetree}; EncryptionMix encmix; + Relation rel; foreach(lcmd, stmt->cmds) { @@ -380,17 +381,25 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) errmsg("Recursive ALTER TABLE on a mix of encrypted and unencrypted relations is not supported")); } + rel = relation_open(relid, NoLock); + /* * With a SET ACCESS METHOD clause, use that as the basis for * decisions. But if it's not present, look up encryption status * of the table. + * + * Since partitioned tables lack storage we do not need to set the + * encryption mode. */ - if (setAccessMethod) + if (setAccessMethod && RELKIND_HAS_STORAGE(rel->rd_rel->relkind)) { event.rebuildSequencesFor = relid; if (shouldEncryptTable(setAccessMethod->name)) + { event.encryptMode = TDE_ENCRYPT_MODE_ENCRYPT; + checkPrincipalKeyConfigured(); + } else event.encryptMode = TDE_ENCRYPT_MODE_PLAIN; } @@ -405,6 +414,8 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) event.encryptMode = TDE_ENCRYPT_MODE_PLAIN; } + relation_close(rel, NoLock); + push_event_stack(&event); checkEncryptionStatus(); } From 8b1f1cf398ebe1d3e7b640d96e8bd6f6ad517790 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Wed, 18 Jun 2025 17:56:33 +0200 Subject: [PATCH 04/21] Try to use poll_start instead of kill9_until_dead in TAP tests The poll_start function handles both slow kill -9 and slow startup while kill9_until_dead only really handles one of them. --- contrib/pg_tde/t/crash_recovery.pl | 16 ++++++++-------- contrib/pg_tde/t/pgtde.pm | 28 +++++++++++++++++++--------- contrib/pg_tde/t/replication.pl | 4 ++-- contrib/pg_tde/t/unlogged_tables.pl | 4 ++-- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/contrib/pg_tde/t/crash_recovery.pl b/contrib/pg_tde/t/crash_recovery.pl index 65bae54751666..4fa2ea2d3ef3f 100644 --- a/contrib/pg_tde/t/crash_recovery.pl +++ b/contrib/pg_tde/t/crash_recovery.pl @@ -51,10 +51,10 @@ PGTDE::psql($node, 'postgres', "ALTER SYSTEM SET pg_tde.wal_encrypt = 'on';"); PGTDE::append_to_result_file("-- kill -9"); -PGTDE::kill9_until_dead($node); +$node->kill9; PGTDE::append_to_result_file("-- server start"); -$node->start; +PGTDE::poll_start($node); PGTDE::append_to_result_file("-- rotate wal key"); PGTDE::psql($node, 'postgres', @@ -71,12 +71,12 @@ ); PGTDE::psql($node, 'postgres', "INSERT INTO test_enc (x) VALUES (3), (4);"); PGTDE::append_to_result_file("-- kill -9"); -PGTDE::kill9_until_dead($node); +$node->kill9; PGTDE::append_to_result_file("-- server start"); PGTDE::append_to_result_file( "-- check that pg_tde_save_principal_key_redo hasn't destroyed a WAL key created during the server start" ); -$node->start; +PGTDE::poll_start($node); PGTDE::append_to_result_file("-- rotate wal key"); PGTDE::psql($node, 'postgres', @@ -93,24 +93,24 @@ ); PGTDE::psql($node, 'postgres', "INSERT INTO test_enc (x) VALUES (5), (6);"); PGTDE::append_to_result_file("-- kill -9"); -PGTDE::kill9_until_dead($node); +$node->kill9; PGTDE::append_to_result_file("-- server start"); PGTDE::append_to_result_file( "-- check that the key rotation hasn't destroyed a WAL key created during the server start" ); -$node->start; +PGTDE::poll_start($node); PGTDE::psql($node, 'postgres', "TABLE test_enc;"); PGTDE::psql($node, 'postgres', "CREATE TABLE test_enc2 (x int PRIMARY KEY) USING tde_heap;"); PGTDE::append_to_result_file("-- kill -9"); -PGTDE::kill9_until_dead($node); +$node->kill9; PGTDE::append_to_result_file("-- server start"); PGTDE::append_to_result_file( "-- check redo of the smgr internal key creation when the key is on disk" ); -$node->start; +PGTDE::poll_start($node); $node->stop; diff --git a/contrib/pg_tde/t/pgtde.pm b/contrib/pg_tde/t/pgtde.pm index 3b5bb57f6c777..eb5c02b24ec17 100644 --- a/contrib/pg_tde/t/pgtde.pm +++ b/contrib/pg_tde/t/pgtde.pm @@ -37,22 +37,32 @@ sub psql } } -sub kill9_until_dead +# Copied from src/test/recovery/t/017_shm.pl +sub poll_start { my ($node) = @_; - return unless defined $node->{_pid}; # Cluster already stopped + my $max_attempts = 10 * $PostgreSQL::Test::Utils::timeout_default; + my $attempts = 0; - my $pid = $node->{_pid}; - $node->kill9; - - # Wait for process to actually die. - while (kill(0, $pid) != 0) + while ($attempts < $max_attempts) { - sleep(0.1); + $node->start(fail_ok => 1) && return 1; + + # Wait 0.1 second before retrying. + usleep(100_000); + + # Clean up in case the start attempt just timed out or some such. + $node->stop('fast', fail_ok => 1); + + $attempts++; } -} + # Try one last time without fail_ok, which will BAIL_OUT unless it + # succeeds. + $node->start && return 1; + return 0; +} sub append_to_result_file { diff --git a/contrib/pg_tde/t/replication.pl b/contrib/pg_tde/t/replication.pl index 573ef3e8050cf..14c3af1c2f2f2 100644 --- a/contrib/pg_tde/t/replication.pl +++ b/contrib/pg_tde/t/replication.pl @@ -82,10 +82,10 @@ PGTDE::psql($primary, 'postgres', "ALTER SYSTEM SET pg_tde.wal_encrypt = 'on';"); -PGTDE::kill9_until_dead($primary); +$primary->kill9; PGTDE::append_to_result_file("-- primary start"); -$primary->start; +PGTDE::poll_start($primary); $primary->wait_for_catchup('replica'); PGTDE::psql($replica, 'postgres', "SELECT * FROM test_enc2 ORDER BY x;"); diff --git a/contrib/pg_tde/t/unlogged_tables.pl b/contrib/pg_tde/t/unlogged_tables.pl index 365f89c8fa0c9..e348ebaef5466 100644 --- a/contrib/pg_tde/t/unlogged_tables.pl +++ b/contrib/pg_tde/t/unlogged_tables.pl @@ -35,10 +35,10 @@ PGTDE::psql($node, 'postgres', "CHECKPOINT;"); PGTDE::append_to_result_file("-- kill -9"); -PGTDE::kill9_until_dead($node); +$node->kill9; PGTDE::append_to_result_file("-- server start"); -$node->start; +PGTDE::poll_start($node); PGTDE::psql($node, 'postgres', "TABLE t;"); From 98c310907241f01d3fa23574b8eaa919749b3e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85strand?= Date: Mon, 23 Jun 2025 11:07:28 +0200 Subject: [PATCH 05/21] Remove extra word in error message for existing key This "to" had no business being in this message. --- contrib/pg_tde/expected/key_provider.out | 4 ++-- contrib/pg_tde/src/catalog/tde_principal_key.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/pg_tde/expected/key_provider.out b/contrib/pg_tde/expected/key_provider.out index 8110100372452..e2a018c606fc0 100644 --- a/contrib/pg_tde/expected/key_provider.out +++ b/contrib/pg_tde/expected/key_provider.out @@ -351,9 +351,9 @@ SELECT pg_tde_create_key_using_database_key_provider('existing-key','file-provid (1 row) SELECT pg_tde_create_key_using_database_key_provider('existing-key','file-provider'); -ERROR: cannot to create key "existing-key" because it already exists +ERROR: cannot create key "existing-key" because it already exists SELECT pg_tde_create_key_using_global_key_provider('existing-key','file-keyring'); -ERROR: cannot to create key "existing-key" because it already exists +ERROR: cannot create key "existing-key" because it already exists -- Setting principal key fails if key does not exist SELECT pg_tde_set_default_key_using_global_key_provider('not-existing', 'file-keyring'); ERROR: key "not-existing" does not exist diff --git a/contrib/pg_tde/src/catalog/tde_principal_key.c b/contrib/pg_tde/src/catalog/tde_principal_key.c index 3944d123825db..6d0df4c641ea8 100644 --- a/contrib/pg_tde/src/catalog/tde_principal_key.c +++ b/contrib/pg_tde/src/catalog/tde_principal_key.c @@ -519,7 +519,7 @@ pg_tde_create_principal_key_internal(Oid providerOid, if (key_info != NULL) ereport(ERROR, errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("cannot to create key \"%s\" because it already exists", key_name)); + errmsg("cannot create key \"%s\" because it already exists", key_name)); key_info = KeyringGenerateNewKeyAndStore(provider, key_name, PRINCIPAL_KEY_LEN); From ef03f7b9fe528a4db10e339d3adb39004aef199c Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Wed, 25 Jun 2025 14:46:22 +0300 Subject: [PATCH 06/21] Create enforcement.md - Encryption Enforcement topic (#403) Wrote a general topic regarding Encrpytion Enforcement from pg_tde perspective. --- .../documentation/docs/how-to/enforcement.md | 65 +++++++++++++++++++ .../pg_tde/documentation/docs/variables.md | 26 ++++---- contrib/pg_tde/documentation/mkdocs.yml | 1 + 3 files changed, 79 insertions(+), 13 deletions(-) create mode 100644 contrib/pg_tde/documentation/docs/how-to/enforcement.md diff --git a/contrib/pg_tde/documentation/docs/how-to/enforcement.md b/contrib/pg_tde/documentation/docs/how-to/enforcement.md new file mode 100644 index 0000000000000..7796b7ef809b4 --- /dev/null +++ b/contrib/pg_tde/documentation/docs/how-to/enforcement.md @@ -0,0 +1,65 @@ +# Encryption Enforcement + +For `pg_tde`, encryption enforcement ensures that only encrypted storage is allowed for specific operations, tables, or the entire database. It prevents the accidental creation of unencrypted tables or indexes in environments where encryption is required for compliance, security, or policy enforcement. + +## What does enforcement do? + +When enabled, encryption enforcement: + +* Prevents creation of unencrypted tables or indexes +* Enforces consistent encryption usage across tenants, databases, or users +* Can be scoped globally, per database, or per role + +## Enforce encryption usage + +Use the following techniques to enforce the secure use of `pg_tde`. + +### 1. Enforce encryption across the server + +To enforce encryption cluster-wide, set the [`pg_tde.enforce_encryption`](../variables.md/#pg_tdeenforce_encryption) variable in `postgresql.conf`: + +```ini +pg_tde.enforce_encryption = on +``` + +!!! note + **Only** superusers can set or change this variable. + +This ensures that no user, including superusers, can create unencrypted tables. Superusers can however explicitly [override the variable in their session](#override-enforcement-for-trusted-sessions). + +### 2. Enforce encryption for a specific database + +To apply encryption enforcement only within a specific database, run: + +```sql +ALTER DATABASE example_db SET pg_tde.enforce_encryption = on; +``` + +This ensures encryption is enforced **only** when connected to that database. + +### 3. Enforce encryption for a specific user + +You can also enforce encryption on a per-user basis, run: + +```sql +ALTER USER example_user SET pg_tde.enforce_encryption = on; +``` + +This ensures that the user `example_user` cannot create unencrypted tables, regardless of which database they connect to. + +### Override enforcement for trusted sessions + +Superusers can override the variable at the session level: + +```sql +SET pg_tde.enforce_encryption = off; +``` + +This allows temporary creation of unencrypted tables in special cases, such as: + +* Loading trusted, public reference datasets +* Benchmarking and test environments +* Migration staging before re-encryption + +!!! note + While superusers can disable enforcement in their session, they must do so explicitly. Enforcement defaults remain active to protect from accidental misconfiguration. diff --git a/contrib/pg_tde/documentation/docs/variables.md b/contrib/pg_tde/documentation/docs/variables.md index 9947eacc385fe..46d27730857fa 100644 --- a/contrib/pg_tde/documentation/docs/variables.md +++ b/contrib/pg_tde/documentation/docs/variables.md @@ -32,14 +32,14 @@ Similarly, `ALTER TABLE SET ACCESS METHOD` is only allowed, if the access me Other DDL operations are still allowed. For example other `ALTER` commands are allowed on unencrypted tables, as long as the access method isn't changed. -You can set this variable at the following levels: +You can set this variable at the following levels: -* global - for the entire PostgreSQL cluster. -* database - for specific databases. -* user - for specific users. -* session - for the current session. +* global - for the entire PostgreSQL cluster +* database - for specific databases +* user - for specific users +* session - for the current session -Setting or changing the value requires superuser permissions. +Setting or changing the value requires superuser permissions. For examples, see the [Encryption Enforcement](how-to/enforcement.md) topic. ## pg_tde.inherit_global_providers @@ -52,12 +52,12 @@ If disabled, functions that change the key providers can only work with database In this case, the default principal key, if set, is also disabled. -You can set this variable at the following levels: +You can set this variable at the following levels: -* global - for the entire PostgreSQL cluster. -* database - for specific databases. -* user - for specific users. -* session - for the current session. +* global - for the entire PostgreSQL cluster +* database - for specific databases +* user - for specific users +* session - for the current session - -Setting this variable doesn't affect existing uses of global keys. It only prevents the creation of new principal keys using global providers. +!!! note + Setting this variable doesn't affect existing uses of global keys. It only prevents the creation of new principal keys using global providers. diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index 9d542f2156a1c..72ae254c60a19 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -198,6 +198,7 @@ nav: - "pg_checksums": command-line-tools/pg-tde-checksums.md - "Uninstall pg_tde": how-to/uninstall.md - "Configure Multi-tenancy": how-to/multi-tenant-setup.md + - "Encryption Enforcement": how-to/enforcement.md - "Decrypt an Encrypted Table": how-to/decrypt.md - faq.md - "Release Notes": From 2a1f301e627cc25008b220e01397707835777c96 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Wed, 25 Jun 2025 16:27:11 +0300 Subject: [PATCH 07/21] Created replication.md quick walkthrough for pg_tde (#319) Created new replication topic which outlines how to set up PostgreSQL streaming replication when the `pg_tde` extension, specifically the `tde_heap` access method, is enabled on the primary server. --- .../pg_tde/documentation/docs/replication.md | 105 +++++++++++++++++- contrib/pg_tde/documentation/mkdocs.yml | 1 + 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/replication.md b/contrib/pg_tde/documentation/docs/replication.md index b745107a2f222..88af22ba04dde 100644 --- a/contrib/pg_tde/documentation/docs/replication.md +++ b/contrib/pg_tde/documentation/docs/replication.md @@ -1,4 +1,103 @@ -# Replication +# Streaming Replication with tde_heap - ## Are my backups safe? Can I restore from them? @@ -162,3 +167,7 @@ To restore from an encrypted backup, you must have the same principal encryption ## I'm using OpenSSL in FIPS mode and need to use pg_tde. Does pg_tde comply with FIPS requirements? Can I use my own FIPS-mode OpenSSL library with pg_tde? Yes. `pg_tde` works with the FIPS-compliant version of OpenSSL, whether it is provided by your operating system or if you use your own OpenSSL libraries. If you use your own libraries, make sure they are FIPS certified. + +## Is post-quantum encryption supported? + +No. Post-quantum encryption is not currently supported. diff --git a/contrib/pg_tde/documentation/docs/wal-encryption.md b/contrib/pg_tde/documentation/docs/wal-encryption.md index c61692e530e25..68cfa9c65e864 100644 --- a/contrib/pg_tde/documentation/docs/wal-encryption.md +++ b/contrib/pg_tde/documentation/docs/wal-encryption.md @@ -1,7 +1,7 @@ # Configure WAL Encryption (tech preview) !!! warning - The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. + The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. Before enabling WAL encryption, follow the steps below to create a principal key and configure it for WAL: From 5a2c081a959a5a763ceb02b2e2acc395078266e3 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Thu, 26 Jun 2025 16:32:49 +0200 Subject: [PATCH 13/21] PG-1257 Add key deletion funcs to documentation Add principal key deletion functions to documentation. Fix couple uncertainties on architecture docs page. --- .../documentation/docs/architecture/index.md | 4 ++-- contrib/pg_tde/documentation/docs/functions.md | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/architecture/index.md b/contrib/pg_tde/documentation/docs/architecture/index.md index 9abd070c4c9eb..54542ee94e0b6 100644 --- a/contrib/pg_tde/documentation/docs/architecture/index.md +++ b/contrib/pg_tde/documentation/docs/architecture/index.md @@ -305,12 +305,12 @@ You can manage a default key with the following functions: * `pg_tde_delete_default_key()` !!! note - `pg_tde_delete_default_key()` is only possible if there's no table currently using the default principal key. + `pg_tde_delete_default_key()` is only possible if there's no database currently using the default principal key. Changing the default principal key will rotate the encryption of internal keys for all databases using the current default principal key. #### Delete a key -The `pg_tde_delete_key()` function removes the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a global default principal key, internal keys will be encrypted with the default key. +The `pg_tde_delete_key()` function removes the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a default principal key, internal keys will be encrypted with the default key. !!! note WAL keys **cannot** be deleted, as server keys are managed separately. diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 563d359aa2cf4..fcc7d03c51ef4 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -302,6 +302,22 @@ SELECT pg_tde_set_default_key_using_global_key_provider( ); ``` +### pg_tde_delete_key + +Deletes the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a default principal key, internal keys will be encrypted with the default key. + +```sql +SELECT pg_tde_delete_key(); +``` + +### pg_tde_delete_default_key + +Deletes default principal key. It's possible only if no database uses default principal key. + +```sql +SELECT pg_tde_delete_default_key(); +``` + ## Encryption status check ### pg_tde_is_encrypted From aa2357803fddc35330892c975c752cb95ad9d687 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Thu, 26 Jun 2025 19:14:00 +0300 Subject: [PATCH 14/21] Docs 17.5.1 revert commit 56106 (#458) reverted set key changes for architecture, functions, set principal key and multi-tenant-setup.md --- .../documentation/docs/architecture/index.md | 17 ++--- .../pg_tde/documentation/docs/functions.md | 69 ++++++++++--------- .../set-principal-key.md | 27 +++----- .../docs/how-to/multi-tenant-setup.md | 57 ++++----------- 4 files changed, 64 insertions(+), 106 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/architecture/index.md b/contrib/pg_tde/documentation/docs/architecture/index.md index 54542ee94e0b6..f1c424bb7e2a7 100644 --- a/contrib/pg_tde/documentation/docs/architecture/index.md +++ b/contrib/pg_tde/documentation/docs/architecture/index.md @@ -276,19 +276,15 @@ pg_tde_REVOKE_database_key_management_FROM_role ### Creating and rotating keys -Principal keys can be created using the following functions: +Principal keys can be created or rotated using the following functions: ```sql -pg_tde_create_key_using_(global/database)_key_provider('key-name', 'provider-name') +pg_tde_set_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) +pg_tde_set_server_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) +pg_tde_set_default_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) ``` -Principal keys can be used or rotated using the following functions: - -```sql -pg_tde_set_key_using_(global/database)_key_provider('key-name', 'provider-name') -pg_tde_set_server_key_using_(global/database)_key_provider('key-name', 'provider-name') -pg_tde_set_default_key_using_(global/database)_key_provider('key-name', 'provider-name') -``` +`ensure_new_key` is a boolean parameter defaulting to false. If it is `true` the function might return an error instead of setting the key if it already exists on the provider. ### Default principal key @@ -300,8 +296,7 @@ With this feature, it is possible for the entire database server to easily use t You can manage a default key with the following functions: -* `pg_tde_create_key_using_global_key_provider('key-name','provider-name')` -* `pg_tde_set_default_key_using_global_key_provider('key-name','provider-name')` +* `pg_tde_set_default_key_using_global_key_provider('key-name','provider-name','true/false')` * `pg_tde_delete_default_key()` !!! note diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index fcc7d03c51ef4..39f1371fee4d1 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -230,78 +230,85 @@ These functions list the details of all key providers for the current database o ## Principal key management -Use these functions to create a new principal key at a given keyprover, and to use those keys for a specific scope such as a current database, a global or default scope. You can also use them to start using a different existing key for a specific scope. +Use these functions to create a new principal key for a specific scope such as a current database, a global or default scope. You can also use them to start using a different existing key for a specific scope. Princial keys are stored on key providers by the name specified in this function - for example, when using the Vault provider, after creating a key named "foo", a key named "foo" will be visible on the Vault server at the specified mount point. -### pg_tde_creates_key_using_database_key_provider - -Creates a principal key at a database local key provider with the given name. For later use with pg_tde_set_key_using_database_key_provider(). - -```sql -SELECT pg_tde_create_key_using_database_key_provider( - 'key-name', - 'provider-name' -); -``` -### pg_tde_create_key_using_global_key_provider - -Creates a principal key at a global key provider with the given name. For later use with pg_tde_set_ series of functions. - -```sql -SELECT pg_tde_create_key_using_global_key_provider( - 'key-name', - 'provider-name' -); -``` - ### pg_tde_set_key_using_database_key_provider -Sets the principal key for the **current** database, using the specified local key provider. It also rotates internal encryption keys to use the specified principal key. +Creates or reuses a principal key for the **current** database, using the specified local key provider. It also rotates internal encryption keys to use the specified principal key. This function is typically used when working with per-database encryption through a local key provider. ```sql SELECT pg_tde_set_key_using_database_key_provider( 'key-name', - 'provider-name' + 'provider-name', + 'false' -- or 'true' ); ``` + +For the third parameter (`true`, `false`, or omitted): + +* `true`: Requires the key to be newly created. If a key with the same name already exists, the function fails. +* `false` (default if omitted): Reuses the existing key with that name, if present. If the key does not exist, a new key is created. + ### pg_tde_set_key_using_global_key_provider -Sets or rotates the global principal key using the specified global key provider and the key name. This key is used for global settings like WAL encryption. +Creates or rotates the global principal key using the specified global key provider and the key name. This key is used for global settings like WAL encryption. ```sql SELECT pg_tde_set_key_using_global_key_provider( 'key-name', - 'provider-name' + 'provider-name', + 'ensure_new_key' ); ``` + The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: + +* If set to `true`, a new key must be unique. + If the provider already stores a key by that name, the function returns an error. +* If set to `false` (default), an existing principal key may be reused. + ### pg_tde_set_server_key_using_global_key_provider -Sets or rotates the server principal key using the specified global key provider. Use this function to set a principal key for WAL encryption. +Creates or rotates the server principal key using the specified global key provider. Use this function to set a principal key for WAL encryption. ```sql SELECT pg_tde_set_server_key_using_global_key_provider( 'key-name', - 'provider-name' + 'provider-name', + 'ensure_new_key' ); ``` +The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: + +* If set to `true`, a new key must be unique. + If the provider already stores a key by that name, the function returns an error. +* If set to `false` (default), an existing principal key may be reused. + ### pg_tde_set_default_key_using_global_key_provider -Sets or rotates the default principal key for the server using the specified global key provider. +Creates or rotates the default principal key for the server using the specified global key provider. -The default key is automatically used as a principal key by any database that doesn't have an individual key provider and key configuration. +The default key is automatically used as a principal key by any database that doesn't have an individual key provider and key configuration. ```sql SELECT pg_tde_set_default_key_using_global_key_provider( 'key-name', - 'provider-name' + 'provider-name', + 'ensure_new_key' ); ``` +The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: + +* If set to `true`, a new key must be unique. + If the provider already stores a key by that name, the function returns an error. +* If set to `false` (default), an existing principal key may be reused. + ### pg_tde_delete_key Deletes the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a default principal key, internal keys will be encrypted with the default key. diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md index 4c3cc3802d04e..86bb9b1ada853 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md @@ -4,23 +4,13 @@ You can configure a default principal key using a global key provider. This key ## Create a default principal key -To create a global principal key, run: - -```sql -SELECT pg_tde_create_key_using_global_key_provider( - 'key-name', - 'global_vault_provider' -); -``` - -## Configure a default principal key - To configure a global principal key, run: ```sql SELECT pg_tde_set_default_key_using_global_key_provider( 'key-name', - 'global_vault_provider' + 'global_vault_provider', + 'false' -- or 'true', or omit entirely ); ``` @@ -28,10 +18,13 @@ SELECT pg_tde_set_default_key_using_global_key_provider( * `key-name` is the name under which the principal key is stored in the provider. * `global_vault_provider` is the name of the global key provider you previously configured. +* Third parameter (optional): + * `true` requires the key to be newly created. If the key already exists, the function fails. + * `false` or omitted (default), allows reuse of an existing key if it exists. If not, a new key is created under the specified name. ## How key generation works -The key material (actual cryptographic key) is auto-generated by `pg_tde` and stored securely by the configured provider. +If the specified key does **not** exist, a new encryption key is created under the given name. In this case, the key material (actual cryptographic key) is auto-generated by `pg_tde` and stored securely by the configured provider. !!! note This process sets the **default principal key** for the server. Any database without its own key configuration will use this key. @@ -41,14 +34,10 @@ The key material (actual cryptographic key) is auto-generated by `pg_tde` and st This example is for testing purposes only. Replace the key name and provider name with your values: ```sql -SELECT pg_tde_create_key_using_global_key_provider( - 'test-db-master-key', - 'file-vault' -); - SELECT pg_tde_set_key_using_global_key_provider( 'test-db-master-key', - 'file-vault' + 'file-vault', + 'false' ); ``` diff --git a/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md b/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md index 19a9974b3df28..6e9454dded4b5 100644 --- a/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md +++ b/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md @@ -42,7 +42,7 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo !!! tip - You can have the `pg_tde` extension automatically enabled for every newly created database. Modify the template `template1` database as follows: + You can have the `pg_tde` extension automatically enabled for every newly created database. Modify the template `template1` database as follows: ```sh psql -d template1 -c 'CREATE EXTENSION pg_tde;' @@ -59,8 +59,8 @@ You must do these steps for every database where you have created the extension. The KMIP server setup is out of scope of this document. Make sure you have obtained the root certificate for the KMIP server and the keypair for the client. The client key needs permissions to create / read keys on the server. Find the [configuration guidelines for the HashiCorp Vault Enterprise KMIP Secrets Engine](https://developer.hashicorp.com/vault/tutorials/enterprise/kmip-engine). - - For testing purposes, you can use the PyKMIP server which enables you to set up required certificates. To use a real KMIP server, make sure to obtain the valid certificates issued by the key management appliance. + + For testing purposes, you can use the PyKMIP server which enables you to set up required certificates. To use a real KMIP server, make sure to obtain the valid certificates issued by the key management appliance. ```sql SELECT pg_tde_add_database_key_provider_kmip( @@ -100,16 +100,10 @@ You must do these steps for every database where you have created the extension. The Vault server setup is out of scope of this document. ```sql - SELECT pg_tde_add_database_key_provider_vault_v2( - 'provider-name', - 'url', - 'mount', - 'secret_token_path', - 'ca_path' - ); - ``` + SELECT pg_tde_add_database_key_provider_vault_v2('provider-name', 'url', 'mount', 'secret_token_path', 'ca_path'); + ``` - where: + where: * `url` is the URL of the Vault server * `mount` is the mount point where the keyring should store the keys @@ -147,52 +141,25 @@ You must do these steps for every database where you have created the extension. '/tmp/pg_tde_test_local_keyring.per' ); ``` + +2. Add a principal key -2. Create a key ```sql - - SELECT pg_tde_create_key_using_database_key_provider( - 'name-of-the-key', - 'provider-name' - ); + SELECT pg_tde_set_key_using_database_key_provider('name-of-the-key', 'provider-name','ensure_new_key'); ``` where: * `name-of-the-key` is the name of the principal key. You will use this name to identify the key. - * `provider-name` is the name of the key provider you added before. The principal key is associated with this provider and it is the location where it is stored and fetched from. + * `provider-name` is the name of the key provider you added before. The principal key will be associated with this provider. + * `ensure_new_key` defines if a principal key must be unique. The default value `true` means that you must speficy a unique key during key rotation. The `false` value allows reusing an existing principal key. :material-information: Warning: This example is for testing purposes only: ```sql - SELECT pg_tde_create_key_using_database_key_provider( - 'test-db-master-key', - 'file-vault' - ); + SELECT pg_tde_set_key_using_database_key_provider('test-db-master-key','file-vault','ensure_new_key'); ``` !!! note The key is auto-generated. -3. Use the key as principal key - ```sql - - SELECT pg_tde_set_key_using_database_key_provider( - 'name-of-the-key', - 'provider-name' - ); - ``` - - where: - - * `name-of-the-key` is the name of the principal key. You will use this name to identify the key. - * `provider-name` is the name of the key provider you added before. The principal key will be associated with this provider. - - :material-information: Warning: This example is for testing purposes only: - - ```sql - SELECT pg_tde_set_key_using_database_key_provider( - 'test-db-master-key', - 'file-vault' - ); - ``` From 8d88d3f28a060d060db0e1eedb5aa47ce3d1150f Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Fri, 27 Jun 2025 15:26:09 +0300 Subject: [PATCH 15/21] Updated principal-key/features/functions.md based on AA feedback (#441) In set-principal-key.md: * updated with correct code example using set_server_key_using_global parameter * updated note to reflect correct config In features.md: * Removed temporary tables feature to clear confusion, removed logical replication mention, removed WAL encryption as a feature. In functions.md: * Added ON FUNCTION for grant/revoke execution * Modified sensitive info bolded paragraph to important note * Small modifications to notes display, title cases and text fixes * added note to Add or modify Vault providers for keeping the same principal key. * Added warning for WAL in pg_tde_create_key_using_global_key_provider In general: * Removed all logical replication mentions except the FAQ and in RC2 release note. --- contrib/pg_tde/documentation/docs/features.md | 4 +- .../pg_tde/documentation/docs/functions.md | 56 +++++++++++++------ .../set-principal-key.md | 9 ++- .../docs/index/supported-versions.md | 4 +- .../documentation/docs/index/tde-encrypts.md | 3 +- 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/features.md b/contrib/pg_tde/documentation/docs/features.md index e7dc760e1c821..ce9cd10826a81 100644 --- a/contrib/pg_tde/documentation/docs/features.md +++ b/contrib/pg_tde/documentation/docs/features.md @@ -9,16 +9,14 @@ The following features are available for the extension: * Data tables * Index data for encrypted tables * TOAST tables - * Temporary tables created during database operations + * Temporary tables !!! note Metadata of those tables is not encrypted. -* Global Write-Ahead Log (WAL) encryption for data in both encrypted and non-encrypted tables * Single-tenancy support via a global keyring provider * Multi-tenancy support * Table-level granularity for encryption and access control * Multiple Key management options -* Logical replication of encrypted tables [Overview](index/index.md){.md-button} [Get Started](install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 39f1371fee4d1..735c33cfae809 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -8,15 +8,15 @@ By default, `pg_tde` is locked down. No one is allowed to do any operations unti However, database owners can run the “view keys” and “set principal key” functions on their own databases. You can delegate these rights to other roles with the following commands: -* `GRANT EXECUTE` -* `REVOKE EXECUTE` +* `GRANT EXECUTE ON FUNCTION` +* `REVOKE EXECUTE ON FUNCTION` ## Key provider management A key provider is a system or service responsible for managing encryption keys. `pg_tde` supports the following key providers: * local file (not recommended for production use) -* Hashicorp Vault / OpenBao +* HashiCorp Vault / OpenBao * KMIP compatible providers Key provider management includes the following operations: @@ -52,9 +52,11 @@ The `change` functions require the same parameters as the `add` functions. They Provider specific parameters differ for each implementation. Refer to the respective subsection for details. -**Some provider specific parameters contain sensitive information, such as passwords. Never specify these directly, use the remote configuration option instead.** +!!! note + The updated provider must be able to retrieve the same principal keys as the original configuration. + If the new configuration cannot access existing keys, encrypted data and backups will become unreadable. -#### Adding or modifying Vault providers +#### Add or modify Vault providers The Vault provider connects to a HashiCorp Vault or an OpenBao server, and stores the keys on a key-value store version 2. @@ -106,7 +108,7 @@ where: * `secret_token_path` is a path to the file that contains an access token with read and write access to the above mount point * **[optional]** `ca_path` is the path of the CA file used for SSL verification -#### Adding or modifying KMIP providers +#### Add or modify KMIP providers The KMIP provider uses a remote KMIP server. @@ -165,16 +167,16 @@ where: !!! note The specified access parameters require permission to read and write keys at the server. -### Adding or modifying local keyfile providers +### Add or modify local key file providers -This provider manages database keys using a local keyfile. +This provider manages database keys using a local key file. This function is intended for development or quick testing, and stores the keys unencrypted in the specified data file. !!! important - Local keyfile providers are **not recommended** for production environments, they lack the security and manageability of external key management systems. + Local key file providers are **not recommended** for production environments, they lack the security and manageability of external key management systems. -Add a local keyfile provider: +Add a local key file provider: ```sql SELECT pg_tde_add_database_key_provider_file( @@ -188,7 +190,7 @@ SELECT pg_tde_add_global_key_provider_file( ); ``` -Change a local keyfile provider: +Change a local key file provider: ```sql SELECT pg_tde_change_database_key_provider_file( @@ -225,14 +227,33 @@ These functions list the details of all key providers for the current database o * `pg_tde_list_all_database_key_providers()` * `pg_tde_list_all_global_key_providers()` -!!! important - All configuration values include possibly sensitive values, such as passwords. **Never** specify these directly, use the remote configuration option instead. - ## Principal key management -Use these functions to create a new principal key for a specific scope such as a current database, a global or default scope. You can also use them to start using a different existing key for a specific scope. +Use these functions to create a new principal key at a given key provider, and to use those keys for a specific scope such as a current database, a global or default scope. You can also use them to start using a different existing key for a specific scope. + +Principal keys are stored on key providers by the name specified in this function - for example, when using the Vault provider, after creating a key named "foo", a key named "foo" will be visible on the Vault server at the specified mount point. + +### pg_tde_creates_key_using_database_key_provider -Princial keys are stored on key providers by the name specified in this function - for example, when using the Vault provider, after creating a key named "foo", a key named "foo" will be visible on the Vault server at the specified mount point. +Creates a principal key using the database-local key provider with the specified name. Use this key later with [`pg_tde_set_key_using_database_key_provider()`](#pg_tde_set_key_using_database_key_provider). + +```sql +SELECT pg_tde_create_key_using_database_key_provider( + 'key-name', + 'provider-name' +); +``` + +### pg_tde_create_key_using_global_key_provider + +Creates a principal key at a global key provider with the given name. Use this key later with the `pg_tde_set_` series of functions. + +```sql +SELECT pg_tde_create_key_using_global_key_provider( + 'key-name', + 'provider-name' +); +``` ### pg_tde_set_key_using_database_key_provider @@ -283,6 +304,9 @@ SELECT pg_tde_set_server_key_using_global_key_provider( ); ``` +!!! warning + The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. +======= The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: * If set to `true`, a new key must be unique. diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md index 86bb9b1ada853..347dc9dff9094 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md @@ -27,17 +27,16 @@ SELECT pg_tde_set_default_key_using_global_key_provider( If the specified key does **not** exist, a new encryption key is created under the given name. In this case, the key material (actual cryptographic key) is auto-generated by `pg_tde` and stored securely by the configured provider. !!! note - This process sets the **default principal key** for the server. Any database without its own key configuration will use this key. + This process sets the **default principal key for the entire server**. Any database without a key explicitly configured will fall back to this key. ## Example This example is for testing purposes only. Replace the key name and provider name with your values: ```sql -SELECT pg_tde_set_key_using_global_key_provider( - 'test-db-master-key', - 'file-vault', - 'false' +SELECT pg_tde_set_server_key_using_global_key_provider( + 'key-name', + 'provider-name' ); ``` diff --git a/contrib/pg_tde/documentation/docs/index/supported-versions.md b/contrib/pg_tde/documentation/docs/index/supported-versions.md index ed4758d0b671c..d943451450d58 100644 --- a/contrib/pg_tde/documentation/docs/index/supported-versions.md +++ b/contrib/pg_tde/documentation/docs/index/supported-versions.md @@ -1,6 +1,6 @@ # Versions and Supported PostgreSQL Deployments -The `pg_tde` extension is available for [Percona Server for PostgreSQL 17.x](https://docs.percona.com/postgresql/17/postgresql-server.html), an open source, drop-in replacement for PostgreSQL Community. This version provides the `tde_heap` access method and offers [full encryption capabilities](../features.md), including encryption of tables, indexes, WAL data, and support for logical replication. +The `pg_tde` extension is available for [Percona Server for PostgreSQL 17.x](https://docs.percona.com/postgresql/17/postgresql-server.html), an open source, drop-in replacement for PostgreSQL Community. This version provides the `tde_heap` access method and offers [full encryption capabilities](../features.md), including encryption of tables, indexes and WAL data. The extension is tightly integrated with Percona Server for PostgreSQL to deliver enhanced encryption functionality that is not available in community builds. @@ -8,7 +8,7 @@ The extension is tightly integrated with Percona Server for PostgreSQL to delive By using our PostgreSQL distribution, you get: -- **Full encryption support** through the `tde_heap` access method, including tables, indexes, WAL data, and logical replication. +- **Full encryption support** through the `tde_heap` access method, including tables, indexes and WAL data. - **Enhanced performance and enterprise-ready features** not available in community builds. - **Regular updates and security patches** backed by Percona’s expert support team. - **Professional support** and guidance for secure PostgreSQL deployments. diff --git a/contrib/pg_tde/documentation/docs/index/tde-encrypts.md b/contrib/pg_tde/documentation/docs/index/tde-encrypts.md index 53bbb82b2198f..1bc2c70cef3e4 100644 --- a/contrib/pg_tde/documentation/docs/index/tde-encrypts.md +++ b/contrib/pg_tde/documentation/docs/index/tde-encrypts.md @@ -6,6 +6,7 @@ * **Temporary tables** created during the query execution, for data tables created using the extension. * **Write-Ahead Log (WAL) data** for the entire database cluster. This includes WAL data in encrypted and non-encrypted tables. * **Indexes** on encrypted tables. -* **Logical replication data** for encrypted tables (ensures encrypted content is preserved across replicas). + +!!! [Table Access Methods and TDE](table-access-method.md){.md-button} From 58153f9f23b932a438638d0114c0c63579568779 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Fri, 27 Jun 2025 15:43:51 +0300 Subject: [PATCH 16/21] Add OpenBao Topic ver 2 (#459) - added openbao topic and toc update for new file - content based on vault.md descriptions --- .../kmip-openbao.md | 48 +++++++++++++++++++ contrib/pg_tde/documentation/mkdocs.yml | 1 + 2 files changed, 49 insertions(+) create mode 100644 contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-openbao.md diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-openbao.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-openbao.md new file mode 100644 index 0000000000000..7b11a694a3d26 --- /dev/null +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-openbao.md @@ -0,0 +1,48 @@ +# Using OpenBao as a Key Provider + +You can configure `pg_tde` to use OpenBao as a global key provider for managing encryption keys securely. + +!!! note + This guide assumes that your OpenBao server is already set up and accessible. OpenBao configuration is outside the scope of this document, see [OpenBao's official documentation](https://openbao.org/docs/) for more information. + +## Example usage + +To register an OpenBao server as a global key provider: + +```sql +SELECT pg_tde_add_global_key_provider_vault_v2( + 'provider-name', + 'url', + 'mount', + 'secret_token_path', + 'ca_path' +); +``` + +## Parameter descriptions + +* `provider-name` is the name to identify this key provider +* `secret_token_path` is a path to the file that contains an access token with read and write access to the above mount point +* `url` is the URL of the Vault server +* `mount` is the mount point where the keyring should store the keys +* [optional] `ca_path` is the path of the CA file used for SSL verification + +The following example is for testing purposes only. Use secure tokens and proper SSL validation in production environments: + +```sql +SELECT pg_tde_add_global_key_provider_vault_v2( + 'my-openbao-provider', + 'https://openbao.example.com:8200', + 'secret/data', + '/path/to/token_file', + '/path/to/ca_cert.pem' +); +``` + +For more information on related functions, see the link below: + +[Percona pg_tde Function Reference](../functions.md){.md-button} + +## Next steps + +[Global Principal Key Configuration :material-arrow-right:](set-principal-key.md){.md-button} diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index 3a60bca374459..c5ebac5eda6d8 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -181,6 +181,7 @@ nav: - "Fortanix Configuration": global-key-provider-configuration/kmip-fortanix.md - "Vault Configuration": global-key-provider-configuration/vault.md - "Thales Configuration": global-key-provider-configuration/kmip-thales.md + - "Using OpenBao as a Key Provider": global-key-provider-configuration/kmip-openbao.md - "Keyring File Configuration": global-key-provider-configuration/keyring.md - "2.2 Global Principal Key Configuration": global-key-provider-configuration/set-principal-key.md - "3. Validate Encryption with pg_tde": test.md From f10eae39202e7137523c40ba065ecfad31d71ed3 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Fri, 27 Jun 2025 14:48:52 +0200 Subject: [PATCH 17/21] Clarify key deletion funcs description in docs Key deletion fucntions don't delete anything as keys stored in external key management system. So these functions just remove keys from TDE. --- contrib/pg_tde/documentation/docs/architecture/index.md | 4 ++-- contrib/pg_tde/documentation/docs/functions.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/architecture/index.md b/contrib/pg_tde/documentation/docs/architecture/index.md index f1c424bb7e2a7..957173e72620e 100644 --- a/contrib/pg_tde/documentation/docs/architecture/index.md +++ b/contrib/pg_tde/documentation/docs/architecture/index.md @@ -305,10 +305,10 @@ You can manage a default key with the following functions: #### Delete a key -The `pg_tde_delete_key()` function removes the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a default principal key, internal keys will be encrypted with the default key. +The `pg_tde_delete_key()` function unsets the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a default principal key, internal keys will be encrypted with the default key. !!! note - WAL keys **cannot** be deleted, as server keys are managed separately. + WAL keys **cannot** be unset, as server keys are managed separately. ### Current key details diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 735c33cfae809..ed21ae3a49d9a 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -335,7 +335,7 @@ The `ensure_new_key` parameter instructs the function how to handle a principal ### pg_tde_delete_key -Deletes the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a default principal key, internal keys will be encrypted with the default key. +Unsets the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a default principal key, internal keys will be encrypted with the default key. ```sql SELECT pg_tde_delete_key(); @@ -343,7 +343,7 @@ SELECT pg_tde_delete_key(); ### pg_tde_delete_default_key -Deletes default principal key. It's possible only if no database uses default principal key. +Unsets default principal key. It's possible only if no database uses default principal key. ```sql SELECT pg_tde_delete_default_key(); From 33af938d3b179050c2630a0c3eadbb327d4a7428 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Mon, 30 Jun 2025 10:00:03 +0300 Subject: [PATCH 18/21] Re-apply set key changes: revert of revert commit (#461) reverted the revert for set key changes for architecture, functions, set principal key and multi-tenant-setup.md --- .../documentation/docs/architecture/index.md | 17 ++++-- .../pg_tde/documentation/docs/functions.md | 42 ++++---------- .../set-principal-key.md | 30 +++++++--- .../docs/how-to/multi-tenant-setup.md | 57 +++++++++++++++---- 4 files changed, 87 insertions(+), 59 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/architecture/index.md b/contrib/pg_tde/documentation/docs/architecture/index.md index 957173e72620e..08332e56fe7c6 100644 --- a/contrib/pg_tde/documentation/docs/architecture/index.md +++ b/contrib/pg_tde/documentation/docs/architecture/index.md @@ -276,15 +276,19 @@ pg_tde_REVOKE_database_key_management_FROM_role ### Creating and rotating keys -Principal keys can be created or rotated using the following functions: +Principal keys can be created using the following functions: ```sql -pg_tde_set_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) -pg_tde_set_server_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) -pg_tde_set_default_key_using_(global/database)_key_provider('key-name', 'provider-name', ensure_new_key) +pg_tde_create_key_using_(global/database)_key_provider('key-name', 'provider-name') ``` -`ensure_new_key` is a boolean parameter defaulting to false. If it is `true` the function might return an error instead of setting the key if it already exists on the provider. +Principal keys can be used or rotated using the following functions: + +```sql +pg_tde_set_key_using_(global/database)_key_provider('key-name', 'provider-name') +pg_tde_set_server_key_using_(global/database)_key_provider('key-name', 'provider-name') +pg_tde_set_default_key_using_(global/database)_key_provider('key-name', 'provider-name') +``` ### Default principal key @@ -296,7 +300,8 @@ With this feature, it is possible for the entire database server to easily use t You can manage a default key with the following functions: -* `pg_tde_set_default_key_using_global_key_provider('key-name','provider-name','true/false')` +* `pg_tde_create_key_using_global_key_provider('key-name','provider-name')` +* `pg_tde_set_default_key_using_global_key_provider('key-name','provider-name')` * `pg_tde_delete_default_key()` !!! note diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index ed21ae3a49d9a..410d83e7f4815 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -229,7 +229,7 @@ These functions list the details of all key providers for the current database o ## Principal key management -Use these functions to create a new principal key at a given key provider, and to use those keys for a specific scope such as a current database, a global or default scope. You can also use them to start using a different existing key for a specific scope. +Use these functions to create a new principal key at a given keyprover, and to use those keys for a specific scope such as a current database, a global or default scope. You can also use them to start using a different existing key for a specific scope. Principal keys are stored on key providers by the name specified in this function - for example, when using the Vault provider, after creating a key named "foo", a key named "foo" will be visible on the Vault server at the specified mount point. @@ -257,50 +257,35 @@ SELECT pg_tde_create_key_using_global_key_provider( ### pg_tde_set_key_using_database_key_provider -Creates or reuses a principal key for the **current** database, using the specified local key provider. It also rotates internal encryption keys to use the specified principal key. +Sets the principal key for the **current** database, using the specified local key provider. It also rotates internal encryption keys to use the specified principal key. This function is typically used when working with per-database encryption through a local key provider. ```sql SELECT pg_tde_set_key_using_database_key_provider( 'key-name', - 'provider-name', - 'false' -- or 'true' + 'provider-name' ); ``` - -For the third parameter (`true`, `false`, or omitted): - -* `true`: Requires the key to be newly created. If a key with the same name already exists, the function fails. -* `false` (default if omitted): Reuses the existing key with that name, if present. If the key does not exist, a new key is created. - ### pg_tde_set_key_using_global_key_provider -Creates or rotates the global principal key using the specified global key provider and the key name. This key is used for global settings like WAL encryption. +Sets or rotates the global principal key using the specified global key provider and the key name. This key is used for global settings like WAL encryption. ```sql SELECT pg_tde_set_key_using_global_key_provider( 'key-name', - 'provider-name', - 'ensure_new_key' + 'provider-name' ); ``` - The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: - -* If set to `true`, a new key must be unique. - If the provider already stores a key by that name, the function returns an error. -* If set to `false` (default), an existing principal key may be reused. - ### pg_tde_set_server_key_using_global_key_provider -Creates or rotates the server principal key using the specified global key provider. Use this function to set a principal key for WAL encryption. +Sets or rotates the server principal key using the specified global key provider. Use this function to set a principal key for WAL encryption. ```sql SELECT pg_tde_set_server_key_using_global_key_provider( 'key-name', - 'provider-name', - 'ensure_new_key' + 'provider-name' ); ``` @@ -315,24 +300,17 @@ The `ensure_new_key` parameter instructs the function how to handle a principal ### pg_tde_set_default_key_using_global_key_provider -Creates or rotates the default principal key for the server using the specified global key provider. +Sets or rotates the default principal key for the server using the specified global key provider. -The default key is automatically used as a principal key by any database that doesn't have an individual key provider and key configuration. +The default key is automatically used as a principal key by any database that doesn't have an individual key provider and key configuration. ```sql SELECT pg_tde_set_default_key_using_global_key_provider( 'key-name', - 'provider-name', - 'ensure_new_key' + 'provider-name' ); ``` -The `ensure_new_key` parameter instructs the function how to handle a principal key during key rotation: - -* If set to `true`, a new key must be unique. - If the provider already stores a key by that name, the function returns an error. -* If set to `false` (default), an existing principal key may be reused. - ### pg_tde_delete_key Unsets the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a default principal key, internal keys will be encrypted with the default key. diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md index 347dc9dff9094..c069f44e98209 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/set-principal-key.md @@ -4,13 +4,23 @@ You can configure a default principal key using a global key provider. This key ## Create a default principal key +To create a global principal key, run: + +```sql +SELECT pg_tde_create_key_using_global_key_provider( + 'key-name', + 'global_vault_provider' +); +``` + +## Configure a default principal key + To configure a global principal key, run: ```sql SELECT pg_tde_set_default_key_using_global_key_provider( 'key-name', - 'global_vault_provider', - 'false' -- or 'true', or omit entirely + 'global_vault_provider' ); ``` @@ -18,13 +28,10 @@ SELECT pg_tde_set_default_key_using_global_key_provider( * `key-name` is the name under which the principal key is stored in the provider. * `global_vault_provider` is the name of the global key provider you previously configured. -* Third parameter (optional): - * `true` requires the key to be newly created. If the key already exists, the function fails. - * `false` or omitted (default), allows reuse of an existing key if it exists. If not, a new key is created under the specified name. ## How key generation works -If the specified key does **not** exist, a new encryption key is created under the given name. In this case, the key material (actual cryptographic key) is auto-generated by `pg_tde` and stored securely by the configured provider. +The key material (actual cryptographic key) is auto-generated by `pg_tde` and stored securely by the configured provider. !!! note This process sets the **default principal key for the entire server**. Any database without a key explicitly configured will fall back to this key. @@ -34,9 +41,14 @@ If the specified key does **not** exist, a new encryption key is created under t This example is for testing purposes only. Replace the key name and provider name with your values: ```sql -SELECT pg_tde_set_server_key_using_global_key_provider( - 'key-name', - 'provider-name' +SELECT pg_tde_create_key_using_global_key_provider( + 'test-db-master-key', + 'file-vault' +); + +SELECT pg_tde_set_key_using_global_key_provider( + 'test-db-master-key', + 'file-vault' ); ``` diff --git a/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md b/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md index 6e9454dded4b5..19a9974b3df28 100644 --- a/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md +++ b/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md @@ -42,7 +42,7 @@ Load the `pg_tde` at startup time. The extension requires additional shared memo !!! tip - You can have the `pg_tde` extension automatically enabled for every newly created database. Modify the template `template1` database as follows: + You can have the `pg_tde` extension automatically enabled for every newly created database. Modify the template `template1` database as follows: ```sh psql -d template1 -c 'CREATE EXTENSION pg_tde;' @@ -59,8 +59,8 @@ You must do these steps for every database where you have created the extension. The KMIP server setup is out of scope of this document. Make sure you have obtained the root certificate for the KMIP server and the keypair for the client. The client key needs permissions to create / read keys on the server. Find the [configuration guidelines for the HashiCorp Vault Enterprise KMIP Secrets Engine](https://developer.hashicorp.com/vault/tutorials/enterprise/kmip-engine). - - For testing purposes, you can use the PyKMIP server which enables you to set up required certificates. To use a real KMIP server, make sure to obtain the valid certificates issued by the key management appliance. + + For testing purposes, you can use the PyKMIP server which enables you to set up required certificates. To use a real KMIP server, make sure to obtain the valid certificates issued by the key management appliance. ```sql SELECT pg_tde_add_database_key_provider_kmip( @@ -100,10 +100,16 @@ You must do these steps for every database where you have created the extension. The Vault server setup is out of scope of this document. ```sql - SELECT pg_tde_add_database_key_provider_vault_v2('provider-name', 'url', 'mount', 'secret_token_path', 'ca_path'); - ``` + SELECT pg_tde_add_database_key_provider_vault_v2( + 'provider-name', + 'url', + 'mount', + 'secret_token_path', + 'ca_path' + ); + ``` - where: + where: * `url` is the URL of the Vault server * `mount` is the mount point where the keyring should store the keys @@ -141,25 +147,52 @@ You must do these steps for every database where you have created the extension. '/tmp/pg_tde_test_local_keyring.per' ); ``` - -2. Add a principal key +2. Create a key ```sql - SELECT pg_tde_set_key_using_database_key_provider('name-of-the-key', 'provider-name','ensure_new_key'); + + SELECT pg_tde_create_key_using_database_key_provider( + 'name-of-the-key', + 'provider-name' + ); ``` where: * `name-of-the-key` is the name of the principal key. You will use this name to identify the key. - * `provider-name` is the name of the key provider you added before. The principal key will be associated with this provider. - * `ensure_new_key` defines if a principal key must be unique. The default value `true` means that you must speficy a unique key during key rotation. The `false` value allows reusing an existing principal key. + * `provider-name` is the name of the key provider you added before. The principal key is associated with this provider and it is the location where it is stored and fetched from. :material-information: Warning: This example is for testing purposes only: ```sql - SELECT pg_tde_set_key_using_database_key_provider('test-db-master-key','file-vault','ensure_new_key'); + SELECT pg_tde_create_key_using_database_key_provider( + 'test-db-master-key', + 'file-vault' + ); ``` !!! note The key is auto-generated. +3. Use the key as principal key + ```sql + + SELECT pg_tde_set_key_using_database_key_provider( + 'name-of-the-key', + 'provider-name' + ); + ``` + + where: + + * `name-of-the-key` is the name of the principal key. You will use this name to identify the key. + * `provider-name` is the name of the key provider you added before. The principal key will be associated with this provider. + + :material-information: Warning: This example is for testing purposes only: + + ```sql + SELECT pg_tde_set_key_using_database_key_provider( + 'test-db-master-key', + 'file-vault' + ); + ``` From dfcef9f1d2e5c726f3d948f165e15fdf6dbaae79 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Mon, 30 Jun 2025 15:11:37 +0300 Subject: [PATCH 19/21] Prepare general docs for GA release (#434) Updated the introduction with the proper extension name, updated the intro to reflect this. Removed important note about not meant for production and added the No upgrade warning from previous versions (like RC2) to GA. Updates: * removed the block announcement for RC2 at the top of the HTML page in the intro * Added the warning note before installation begins too. * Updated site name to full name. --- .../_resource/overrides/main.html | 7 ------ .../_resourcepdf/overrides/main.html | 7 ------ .../documentation/docs/architecture/index.md | 23 +------------------ contrib/pg_tde/documentation/docs/faq.md | 6 +++-- .../pg_tde/documentation/docs/functions.md | 9 -------- .../kmip-server.md | 2 +- .../docs/how-to/multi-tenant-setup.md | 14 +++++------ contrib/pg_tde/documentation/docs/index.md | 9 ++++---- contrib/pg_tde/documentation/docs/install.md | 6 ++--- .../docs/templates/pdf_cover_page.tpl | 2 +- contrib/pg_tde/documentation/mkdocs.yml | 2 +- 11 files changed, 23 insertions(+), 64 deletions(-) diff --git a/contrib/pg_tde/documentation/_resource/overrides/main.html b/contrib/pg_tde/documentation/_resource/overrides/main.html index 1fca8c3d06631..3cf5dc8d34ff9 100644 --- a/contrib/pg_tde/documentation/_resource/overrides/main.html +++ b/contrib/pg_tde/documentation/_resource/overrides/main.html @@ -3,13 +3,6 @@ -#} {% extends "base.html" %} -{% block announce %} - This is the Release Candidate 2 (RC2) of Percona Transparent Data Encryption (TDE) extension. -

It is not recommended for production environments at this stage.

-

We encourage you to test it and give your feedback. - This will help us improve the product and make it production-ready faster.

-{% endblock %} - {% block scripts %} {{ super() }} diff --git a/contrib/pg_tde/documentation/_resourcepdf/overrides/main.html b/contrib/pg_tde/documentation/_resourcepdf/overrides/main.html index 1fca8c3d06631..3cf5dc8d34ff9 100644 --- a/contrib/pg_tde/documentation/_resourcepdf/overrides/main.html +++ b/contrib/pg_tde/documentation/_resourcepdf/overrides/main.html @@ -3,13 +3,6 @@ -#} {% extends "base.html" %} -{% block announce %} - This is the Release Candidate 2 (RC2) of Percona Transparent Data Encryption (TDE) extension. -

It is not recommended for production environments at this stage.

-

We encourage you to test it and give your feedback. - This will help us improve the product and make it production-ready faster.

-{% endblock %} - {% block scripts %} {{ super() }} diff --git a/contrib/pg_tde/documentation/docs/architecture/index.md b/contrib/pg_tde/documentation/docs/architecture/index.md index 08332e56fe7c6..87a2e863d4c14 100644 --- a/contrib/pg_tde/documentation/docs/architecture/index.md +++ b/contrib/pg_tde/documentation/docs/architecture/index.md @@ -239,21 +239,17 @@ This is also the reason why it requires a `dbOid` instead of a name, as it has n ### Deleting providers -Providers can be deleted by using the +Providers can be deleted by using the following functions: ```sql pg_tde_delete_database_key_provider(provider_name) pg_tde_delete_global_key_provider(provider_name) ``` -functions. - For database specific providers, the function first checks if the provider is used or not, and the provider is only deleted if it's not used. For global providers, the function checks if the provider is used anywhere, WAL or any specific database, and returns an error if it is. -This somewhat goes against the principle that `pg_tde` should not interact with other databases than the one the user is connected to, but on the other hand, it only does this lookup in the internal `pg_tde` metadata, not in postgres catalogs, so it is a gray zone. Making this check makes more sense than potentially making some databases inaccessible. - ### Listing/querying providers `pg_tde` provides 2 functions to show providers: @@ -263,17 +259,6 @@ This somewhat goes against the principle that `pg_tde` should not interact with These functions return a list of provider names, type and configuration. -### Provider permissions - -`pg_tde` implements access control based on execution rights on the administration functions. - -For keys and providers administration, it provides two pair of functions: - -```sql -pg_tde_GRANT_database_key_management_TO_role -pg_tde_REVOKE_database_key_management_FROM_role -``` - ### Creating and rotating keys Principal keys can be created using the following functions: @@ -325,12 +310,6 @@ The `pg_tde_delete_key()` function unsets the principal key for the current data `pg_tde_verify_key()` checks that the key provider is accessible, that the current principal key can be downloaded from it, and that it is the same as the current key stored in memory - if any of these fail, it reports an appropriate error. -### Key permissions - -Users with management permissions to a specific database `(pg_tde_(grant/revoke)_(global/databse)_key_management_(to/from)_role)` can change the keys for the database, and use the current key functions. This includes creating keys using global providers, if `pg_tde.inherit_global_providers` is enabled. - -Also the `pg_tde_(grant/revoke)_database_key_management_to_role` function deals with only the specific permission for the above function: it allows a user to change the key for the database, but not to modify the provider configuration. - ### Creating encrypted tables To create an encrypted table or modify an existing table to be encrypted, use the following commands: diff --git a/contrib/pg_tde/documentation/docs/faq.md b/contrib/pg_tde/documentation/docs/faq.md index 82a8b536e4d4b..36430b722a1c4 100644 --- a/contrib/pg_tde/documentation/docs/faq.md +++ b/contrib/pg_tde/documentation/docs/faq.md @@ -27,7 +27,7 @@ Using TDE helps you avoid the following risks: If to translate sensitive data to files stored in your database, these are user data in tables, temporary files, WAL files. TDE has you covered encrypting all these files. -`pg_tde` does not encrypt system catalogs yet. This means that statistics data and database metadata are not encrypted. The encryption of system catalogs is planned for future releases. +`pg_tde` does not encrypt system catalogs yet. This means that statistics data and database metadata are not encrypted. ## Will logical replication work with pg_tde? @@ -121,7 +121,9 @@ We advise encrypting the whole database only if all your data is sensitive, like For WAL encryption, AES-CTR-128 is used. -The support of other encryption mechanisms such as AES256 is planned for future releases. Reach out to us with your requirements and usage scenarios of other encryption methods are needed. +## Is post-quantum encryption supported? + +No, it's not yet supported. In our implementation we reply on OpenSSL libraries that don't yet support post-quantum encryption. ## Can I encrypt an existing table? diff --git a/contrib/pg_tde/documentation/docs/functions.md b/contrib/pg_tde/documentation/docs/functions.md index 410d83e7f4815..b5c51ec07cdf0 100644 --- a/contrib/pg_tde/documentation/docs/functions.md +++ b/contrib/pg_tde/documentation/docs/functions.md @@ -2,15 +2,6 @@ The `pg_tde` extension provides functions for managing different aspects of its operation: -## Permission management - -By default, `pg_tde` is locked down. No one is allowed to do any operations until you grant them permissions. Only superusers may add or alter global key providers. - -However, database owners can run the “view keys” and “set principal key” functions on their own databases. You can delegate these rights to other roles with the following commands: - -* `GRANT EXECUTE ON FUNCTION` -* `REVOKE EXECUTE ON FUNCTION` - ## Key provider management A key provider is a system or service responsible for managing encryption keys. `pg_tde` supports the following key providers: diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-server.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-server.md index aea589ffb08fb..46d93e7f31a63 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-server.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/kmip-server.md @@ -15,7 +15,7 @@ For testing purposes, you can use a lightweight PyKMIP server, which enables eas SELECT pg_tde_add_global_key_provider_kmip( 'provider-name', 'kmip-IP', - 5696, + `port`, '/path_to/server_certificate.pem', '/path_to/client_cert.pem', '/path_to/client_key.pem' diff --git a/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md b/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md index 19a9974b3df28..c8eaac1afcfe8 100644 --- a/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md +++ b/contrib/pg_tde/documentation/docs/how-to/multi-tenant-setup.md @@ -64,13 +64,13 @@ You must do these steps for every database where you have created the extension. ```sql SELECT pg_tde_add_database_key_provider_kmip( - 'provider-name', - 'kmip-addr', - 5696, - '/path_to/client_cert.pem', - '/path_to/client_key.pem', - '/path_to/server_certificate.pem' - ); + 'provider-name', + 'kmip-addr', + `port`, + '/path_to/client_cert.pem', + '/path_to/client_key.pem', + '/path_to/server_certificate.pem' + ); ``` where: diff --git a/contrib/pg_tde/documentation/docs/index.md b/contrib/pg_tde/documentation/docs/index.md index 30bf17a431bd2..5f2a091a250ff 100644 --- a/contrib/pg_tde/documentation/docs/index.md +++ b/contrib/pg_tde/documentation/docs/index.md @@ -1,9 +1,10 @@ -# pg_tde Documentation +# Percona Transparent Data Encryption for PostgreSQL documentation -`pg_tde` is the open source, community driven and futureproof PostgreSQL extension that provides Transparent Data Encryption (TDE) to protect data at rest. `pg_tde` ensures that the data stored on disk is encrypted, and that no one can read it without the proper encryption keys, even if they gain access to the physical storage media. +Percona Transparent Data Encryption for PostgreSQL (`pg_tde`) is an open source, community driven and futureproof PostgreSQL extension that provides Transparent Data Encryption (TDE) to protect data at rest. `pg_tde` ensures that the data stored on disk is encrypted, and that no one can read it without the proper encryption keys, even if they gain access to the physical storage media. -!!! important - This is the {{release}} version of the extension and **it is not meant for production use yet**. We encourage you to use it in testing environments and [provide your feedback](https://forums.percona.com/c/postgresql/pg-tde-transparent-data-encryption-tde/82). +!!! warning "No upgrade path from RC to GA" + There is no safe upgrade path from the previous versions, such as Release Candidate 2, to the General Availability (GA) version of `pg_tde`. + We recommend starting with a **clean installation** for GA deployments. Avoid using RC environments in production. [Overview](index/index.md){.md-button} [Get Started](install.md){.md-button} diff --git a/contrib/pg_tde/documentation/docs/install.md b/contrib/pg_tde/documentation/docs/install.md index 11cc03623c32d..273bb899cd29c 100644 --- a/contrib/pg_tde/documentation/docs/install.md +++ b/contrib/pg_tde/documentation/docs/install.md @@ -1,8 +1,8 @@ # Install pg_tde - +!!! warning "No upgrade path from RC to GA" + There is no safe upgrade path from the previous versions, such as Release Candidate 2, to the General Availability (GA) version of `pg_tde`. + We recommend starting with a **clean installation** for GA deployments. Avoid using RC environments in production. To install `pg_tde`, use one of the following methods: diff --git a/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl b/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl index 286838e89fc54..9f4fb22ba71de 100644 --- a/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl +++ b/contrib/pg_tde/documentation/docs/templates/pdf_cover_page.tpl @@ -7,5 +7,5 @@ {% if config.site_description %}

{{ config.site_description }}

{% endif %} -

Release Candidate (2025-03-27)

+

1.0 (2025-06-30)

\ No newline at end of file diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index c5ebac5eda6d8..51e45a2b4a833 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -1,6 +1,6 @@ # MkDocs general configuration -site_name: pg_tde documentation +site_name: Percona Transparent Data Encryption for PostgreSQL site_description: Documentation site_author: Percona LLC copyright: > From b6c130569076306c424f27e66a0bf7b88c6a192f Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Mon, 30 Jun 2025 15:30:52 +0300 Subject: [PATCH 20/21] Update architecture/index.md (#439) Updated the Architecture topic with the following: - New intro detailing the long term tde goals in a paragraph - Updated the ## Typical setup scenarios topic with better writing and improved flow - Added note to WAL Encryption that it is not to be used in prod env - General small fixes to paragraphs, wrongly written words and such --- .../documentation/docs/architecture/index.md | 113 +++++++----------- 1 file changed, 45 insertions(+), 68 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/architecture/index.md b/contrib/pg_tde/documentation/docs/architecture/index.md index 87a2e863d4c14..76bc66e3c9ae4 100644 --- a/contrib/pg_tde/documentation/docs/architecture/index.md +++ b/contrib/pg_tde/documentation/docs/architecture/index.md @@ -2,12 +2,12 @@ `pg_tde` is a **customizable, complete, data at rest encryption extension** for PostgreSQL. -Let's break down what it means. +The following sections break down the key architectural components of this design. **Customizable** means that `pg_tde` aims to support many different use cases: * Encrypting either every table in every database or only some tables in some databases -* Encryption keys can be stored on various external key storage servers including Hashicorp Vault, KMIP servers. +* Encryption keys can be stored on various external key storage servers including HashiCorp Vault and KMIP servers. * Using one key for everything or different keys for different databases * Storing every key on the same key storage, or using different storages for different databases * Handling permissions: who can manage database specific or global permissions, who can create encrypted or not encrypted tables @@ -20,7 +20,7 @@ Let's break down what it means. * Indexes * Sequences * Temporary tables -* Write Ahead Log (WAL) +* Write Ahead Log (WAL), still in beta. **Do not enable this feature in production environments**. **Extension** means that `pg_tde` should be implemented only as an extension, possibly compatible with any PostgreSQL distribution, including the open source community version. This requires changes in the PostgreSQL core to make it more extensible. Therefore, `pg_tde` currently works only with the [Percona Server for PostgreSQL](https://docs.percona.com/postgresql/17/index.html) - a binary replacement of community PostgreSQL and included in Percona Distribution for PostgreSQL. @@ -82,13 +82,16 @@ Later decisions are made using a slightly modified Storage Manager (SMGR) API: w ### WAL encryption +!!! note + The WAL encryption feature is currently in beta and is not effective unless explicitly enabled. It is not yet production ready. **Do not enable this feature in production environments**. + WAL encryption is controlled globally via a global GUC variable, `pg_tde.wal_encrypt`, that requires a server restart. WAL keys also contain the [LSN](https://www.postgresql.org/docs/17/wal-internals.html) of the first WAL write after key creation. This allows `pg_tde` to know which WAL ranges are encrypted or not and with which key. -The setting only controls writes so that only WAL writes are encrypted when WAL encryption is enabled. This means that WAL files can contain both encrypted and unencrpyted data, depending on what the status of this variable was when writing the data. +The setting only controls writes so that only WAL writes are encrypted when WAL encryption is enabled. This means that WAL files can contain both encrypted and unencrypted data, depending on what the status of this variable was when writing the data. -`pg_tde` keeps track of the encryption status of WAL records using internal keys. When the server is restarted it writes a new internal key if WAL encryption is enabled, or if it is disabled and was previously enabled it writes a dummy key signalling that WAL encryption ended. +`pg_tde` keeps track of the encryption status of WAL records using internal keys. When the server is restarted it writes a new internal key if WAL encryption is enabled, or if it is disabled and was previously enabled it writes a dummy key signaling that WAL encryption ended. With this information the WAL reader code can decide if a specific WAL record has to be decrypted or not and which key it should use to decrypt it. @@ -156,7 +159,7 @@ With these details `pg_tde` does the following based on user operations: * Uploads a new principal key to it after this key is created * Retrieves the principal key from the service when it is required for decryption -Retreival of the principal key is cached so it only happens when necessary. +Retrieval of the principal key is cached so it only happens when necessary. ### Key provider management @@ -169,24 +172,23 @@ For such situations, `pg_tde` also provides [command line tools](../command-line ### Sensitive key provider information !!! important - Authentication details for key providers are sensitive and must be protected. Do not store these credentials in the `$PGDATA` directory alongside the database. Instead, ensure they are stored in a secure location with strict file system permissions to prevent unauthorized access. ## User interface -### Setting up pg_tde +### Set up pg_tde -To use `pg_tde`, users are required to: +To get started with `pg_tde`, follow these steps: * Add `pg_tde` to the `shared_preload_libraries` in `postgresql.conf` as this is required for the SMGR extensions * Execute `CREATE EXTENSION pg_tde` in the databases where they want to use encryption * Optionally, enable `pg_tde.wal_encrypt` in `postgresql.conf` -* Optionally, disable `pg_tde.inherit_global_providers` in `postgresql.conf` (enabled by default) +* Optionally, disable `pg_tde.inherit_global_providers` in `postgresql.conf` (it is enabled by default) -### Adding providers +### Add providers -Keyring providers can be added to either the global or to the database specific scope. +You can add keyring providers to either the global or database specific scope. If `pg_tde.inherit_global_providers` is `on`, global providers are visible for all databases, and can be used. If `pg_tde.inherit_global_providers` is `off`, global providers are only used for WAL encryption. @@ -203,7 +205,7 @@ To add a database specific provider: pg_tde_add_database_key_provider_('provider_name', ... details ...) ``` -### Changing providers +### Change providers To change a value of a global provider: @@ -217,13 +219,9 @@ To change a value of a database specific provider: pg_tde_change_database_key_provider_('provider_name', ... details ...) ``` -These functions also allow changing the type of a provider. - -The functions however do not migrate any data. They are expected to be used during infrastructure migration, for example when the address of a server changes. +These functions also allow changing the type of a provider but **do not** migrate any data. They are expected to be used during infrastructure migration, for example when the address of a server changes. -Note that in these functions do not verify the parameters. For that, see `pg_tde_verify_key`. - -### Changing providers from the command line +### Change providers from the command line To change a provider from a command line, `pg_tde` provides the `pg_tde_change_key_provider` command line tool. @@ -233,11 +231,12 @@ This tool work similarly to the above functions, with the following syntax: pg_tde_change_key_provider ... details ... ``` -Note that since this tool is expected to be offline, it bypasses all permission checks! +!!! note + Since this tool is expected to be offline, it bypasses all permission checks. This is also the reason why it requires a `dbOid` instead of a name, as it has no way to process the catalog and look up names. -This is also the reason why it requires a `dbOid` instead of a name, as it has no way to process the catalog and look up names. + This tool does not validate any parameters. -### Deleting providers +### Delete providers Providers can be deleted by using the following functions: @@ -250,7 +249,7 @@ For database specific providers, the function first checks if the provider is us For global providers, the function checks if the provider is used anywhere, WAL or any specific database, and returns an error if it is. -### Listing/querying providers +### List/query providers `pg_tde` provides 2 functions to show providers: @@ -259,7 +258,18 @@ For global providers, the function checks if the provider is used anywhere, WAL These functions return a list of provider names, type and configuration. -### Creating and rotating keys +### Provider permissions + +`pg_tde` implements access control based on execution rights on the administration functions. + +For keys and providers administration, it provides two pair of functions: + +```sql +pg_tde_GRANT_database_key_management_TO_role +pg_tde_REVOKE_database_key_management_FROM_role +``` + +### Create and rotate keys Principal keys can be created using the following functions: @@ -279,7 +289,7 @@ pg_tde_set_default_key_using_(global/database)_key_provider('key-name', 'provide With `pg_tde.inherit_global_key_providers`, it is also possible to set up a default global principal key, which will be used by any database which has the `pg_tde` extension enabled, but doesn't have a database specific principal key configured using `pg_tde_set_key_using_(global/database)_key_provider`. -With this feature, it is possible for the entire database server to easily use the same principal key for all databases, completely disabling multi-tenency. +With this feature, it is possible for the entire database server to easily use the same principal key for all databases, completely disabling multi-tenancy. #### Manage a default key @@ -310,7 +320,13 @@ The `pg_tde_delete_key()` function unsets the principal key for the current data `pg_tde_verify_key()` checks that the key provider is accessible, that the current principal key can be downloaded from it, and that it is the same as the current key stored in memory - if any of these fail, it reports an appropriate error. -### Creating encrypted tables +### Key permissions + +Users with management permissions to a specific database `(pg_tde_(grant/revoke)_(global/databse)_key_management_(to/from)_role)` can change the keys for the database, and use the current key functions. This includes creating keys using global providers, if `pg_tde.inherit_global_providers` is enabled. + +Also the `pg_tde_(grant/revoke)_database_key_management_to_role` function deals with only the specific permission for the above function: it allows a user to change the key for the database, but not to modify the provider configuration. + +### Create encrypted tables To create an encrypted table or modify an existing table to be encrypted, use the following commands: @@ -319,47 +335,8 @@ CREATE TABLE t1(a INT) USING tde_heap; ALTER TABLE t1 SET ACCESS METHOD tde_heap; ``` -### Changing the `pg_tde.inherit_global_keys` setting - -It is possible for users to use `pg_tde` with `inherit_global_keys = on`, refer to global keys / keyrings in databases, and then change this setting to `off`. - -In this case existing references to global providers, or the global default principal key will remain working as before, but new references to the global scope can't be made. - -## Typical setup scenarios - -### Simple "one principal key" encryption - -1. Passing the option from the postgres config file the extension: `shared_preload_libraries=‘pg_tde’` -2. `CREATE EXTENSION pg_tde;` in `template1` -3. Adding a global key provider -4. Adding a default principal key using the same global provider -5. Enable WAL encryption to use the default principal key using `ALTER SYSTEM SET pg_tde.wal_encrypt=‘ON’` -6. Restart the server -7. Optionally: setting the `default_table_access_method` to `tde_heap` so that tables are encrypted by default - -Database users don't need permissions to any of the encryption functions: -encryption is managed by the admins, normal users only have to create tables with encryption, which requires no specific permissions. - -### One key storage, but different keys per database - -1. Installing the extension: `shared_preload_libraries` + `pg_tde.wal_encrypt` -2. `CREATE EXTENSION pg_tde;` in `template1` -3. Adding a global key provider -4. Changing the WAL encryption to use the proper global key provider -5. Giving users that are expected to manage database keys permissions for database specific key management, but not database specific key provider management: - specific databases HAVE to use the global key provider - -Note: setting the `default_table_access_method` to `tde_heap` is possible, but instead of `ALTER SYSTEM` only per database using `ALTER DATABASE`, after a principal key is configured for that specific database. - -Alternatively `ALTER SYSTEM` is possible, but table creation in the database will fail if there's no principal key for the database, that has to be created first. - -### Complete multi tenancy - -1. Installing the extension: `shared_preload_libraries` + `pg_tde.wal_encrypt` (that's not multi tenant currently) -2. `CREATE EXTENSION pg_tde;` in any database -3. Adding a global key provider for WAL -4. Changing the WAL encryption to use the proper global key provider +### Change the pg_tde.inherit_global_keys setting -No default configuration: key providers / principal keys are configured as a per database level, permissions are managed per database +It is possible to use `pg_tde` with `inherit_global_keys = on`, refer to the global keys or keyrings in databases, and then change this setting to `off`. -Same note about `default_table_access_method` as above - but in a multi tenant setup, `ALTER SYSTEM` doesn't make much sense. +In this case, existing references to global providers or the global default principal key keep working as before, but new references to the global scope cannot be made. From 85037c48e06841b4bdc7041a8efb86d0ccd0cdb2 Mon Sep 17 00:00:00 2001 From: Dragos Andriciuc Date: Mon, 30 Jun 2025 19:12:51 +0300 Subject: [PATCH 21/21] Create Release Notes for 1.0 (#432) Added initial files and modifications to include 1.0 release notes to the TOC and variables. Updates: * updated the ToC names to make them in line with style guide * updated variable with new release branch and fixed small release note name * updated ## Release Highlights with topics: * Added tickets * Updated Upgrade considerations --- .../docs/release-notes/release-notes-v1.0.md | 61 +++++++++++++++++++ .../docs/release-notes/release-notes.md | 2 + contrib/pg_tde/documentation/mkdocs.yml | 19 +++--- contrib/pg_tde/documentation/variables.yml | 6 +- 4 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md new file mode 100644 index 0000000000000..96a7ce3b1ea2a --- /dev/null +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes-v1.0.md @@ -0,0 +1,61 @@ +# pg_tde 1.0 ({{date.GA10}}) + +The `pg_tde` by Percona extension brings in [Transparent Data Encryption (TDE)](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. + +[Get Started](../install.md){.md-button} + +## Release Highlights + +* **`pg_tde` 1.0 is now GA (Generally Available)** + +And **stable** for encrypting relational data in PostgreSQL using [Transparent Data Encryption (TDE)](../index/index.md). This milestone brings production-level data protection to PostgreSQL workloads. + +* **WAL encryption is still in Beta** + +The WAL encryption feature is currently still in beta and is not effective unless explicitly enabled. **It is not yet production ready.** Do **not** enable this feature in production environments. + +## Upgrade considerations + +`pg_tde` {{tdeversion}} is **not** backward compatible with previous `pg_tde` versions, like Release Candidate 2, due to significant changes in code. This means you **cannot** directly upgrade from one version to another. You must do **a clean installation** of `pg_tde`. + +## Known issues + +* The default `mlock` limit on Rocky Linux 8 for ARM64-based architectures equals the memory page size and is 64 Kb. This results in the child process with `pg_tde` failing to allocate another memory page because the max memory limit is reached by the parent process. + +To prevent this, you can change the `mlock` limit to be at least twice bigger than the memory page size: + +* temporarily for the current session using the `ulimit -l ` command. +* set a new hard limit in the `/etc/security/limits.conf` file. To do so, you require the superuser privileges. + +Adjust the limits with caution since it affects other processes running in your system. + +## Changelog + +### New Features + +- [PG-1257](https://perconadev.atlassian.net/browse/PG-1257) – Added SQL function to remove the current principal key + +### Improvements + +- [PG-1617](https://perconadev.atlassian.net/browse/PG-1617) – Removed relation key cache +- [PG-1635](https://perconadev.atlassian.net/browse/PG-1635) – User-facing TDE functions now return void +- [PG-1605](https://perconadev.atlassian.net/browse/PG-1605) – Removed undeclared dependencies for `pg_tde_grant_database_key_management_to_role()` + +### Bugs Fixed + +- [PG-1581](https://perconadev.atlassian.net/browse/PG-1581) – Fixed PostgreSQL crashes on table access when KMIP key is unavailable after restart +- [PG-1583](https://perconadev.atlassian.net/browse/PG-1583) – Fixed a crash when dropping the `pg_tde` extension with CASCADE after changing the key provider file +- [PG-1585](https://perconadev.atlassian.net/browse/PG-1585) – Fixed the vault provider re-addition that failed after server restart with a new token +- [PG-1592](https://perconadev.atlassian.net/browse/PG-1592) – Improve error logs when Server Key Info is requested without being created +- [PG-1593](https://perconadev.atlassian.net/browse/PG-1593) – Fixed runtime failures when invalid Vault tokens are allowed during key provider creation +- [PG-1600](https://perconadev.atlassian.net/browse/PG-1600) – Fixed Postmaster error when dropping a table with an unavailable key provider +- [PG-1606](https://perconadev.atlassian.net/browse/PG-1606) – Fixed missing superuser check in role grant function leads to misleading errors +- [PG-1607](https://perconadev.atlassian.net/browse/PG-1607) – Improved CA parameter order and surrounding documentation for clearer interpretation +- [PG-1608](https://perconadev.atlassian.net/browse/PG-1608) – Updated and fixed global key configuration parameters in documentation +- [PG-1613](https://perconadev.atlassian.net/browse/PG-1613) – Tested and improved the `pg_tde_change_key_provider` CLI utility +- [PG-1637](https://perconadev.atlassian.net/browse/PG-1637) – Fixed unused keys in key files which caused issues after OID wraparound +- [PG-1651](https://perconadev.atlassian.net/browse/PG-1651) – Fixed the CLI tool when working with Vault key export/import +- [PG-1652](https://perconadev.atlassian.net/browse/PG-1652) – Fixed when the server fails to find encryption keys after CLI-based provider change +- [PG-1662](https://perconadev.atlassian.net/browse/PG-1662) – Fixed the creation of inconsistent encryption status when altering partitioned tables +- [PG-1663](https://perconadev.atlassian.net/browse/PG-1663) – Fixed the indexes on partitioned tables which were not encrypted +- [PG-1700](https://perconadev.atlassian.net/browse/PG-1700) – Fixed the error hint when the principal key is missing diff --git a/contrib/pg_tde/documentation/docs/release-notes/release-notes.md b/contrib/pg_tde/documentation/docs/release-notes/release-notes.md index f5306753eaa77..7a5175285dc82 100644 --- a/contrib/pg_tde/documentation/docs/release-notes/release-notes.md +++ b/contrib/pg_tde/documentation/docs/release-notes/release-notes.md @@ -2,6 +2,8 @@ `pg_tde` extension brings in [Transparent Data Encryption (TDE)](../index/index.md) to PostgreSQL and enables you to keep sensitive data safe and secure. +* [Percona Transparent Database Encryption for PostgreSQL 1.0 ({{date.GA10}})](release-notes-v1.0.md) +* [pg_tde Release Candidate 2 (RC2) ({{date.RC2}})](rc2.md) * [pg_tde Release Candidate 2 (RC2) ({{date.RC2}})](rc2.md) * [pg_tde Release Candidate ({{date.RC}})](rc.md) * [pg_tde Beta2 (2024-12-16)](beta2.md) diff --git a/contrib/pg_tde/documentation/mkdocs.yml b/contrib/pg_tde/documentation/mkdocs.yml index 51e45a2b4a833..eca6171c87aab 100644 --- a/contrib/pg_tde/documentation/mkdocs.yml +++ b/contrib/pg_tde/documentation/mkdocs.yml @@ -163,14 +163,14 @@ nav: - "Features": features.md - "Overview": - "What is Transparent Data Encryption (TDE)?": - - "TDE Overview": index/index.md - - "TDE Benefits": index/how-tde-helps.md - - "How TDE Works": index/how-does-tde-work.md - - "Encrypted Data Scope": index/tde-encrypts.md - - "Table Access Methods and TDE": index/table-access-method.md + - "TDE overview": index/index.md + - "TDE benefits": index/how-tde-helps.md + - "How TDE works": index/how-does-tde-work.md + - "Encrypted data scope": index/tde-encrypts.md + - "Table access methods and TDE": index/table-access-method.md - "Limitations of TDE": index/tde-limitations.md - - "Versions and Supported PostgreSQL Deployments": index/supported-versions.md - - "Get Started": + - "Versions and supported PostgreSQL deployments": index/supported-versions.md + - "Get started": - "1. Install pg_tde": install.md - "1.1 Via apt": apt.md - "1.2 Via yum": yum.md @@ -189,7 +189,7 @@ nav: - "Technical Reference": - "Overview": advanced-topics/index.md - "Architecture": architecture/index.md - - "GUC Variables": variables.md + - "GUC variables": variables.md - "Functions": functions.md - "Streaming Replication with tde_heap": replication.md - "TDE Operations": @@ -204,8 +204,9 @@ nav: - "Decrypt an Encrypted Table": how-to/decrypt.md - "Restore an encrypted pg_tde backup": how-to/restore-backups.md - faq.md - - "Release Notes": + - "Release notes": - "pg_tde release notes": release-notes/release-notes.md + - release-notes/release-notes-v1.0.md - release-notes/rc2.md - release-notes/rc.md - release-notes/beta2.md diff --git a/contrib/pg_tde/documentation/variables.yml b/contrib/pg_tde/documentation/variables.yml index 9d4aa480f8364..2938e44a53a96 100644 --- a/contrib/pg_tde/documentation/variables.yml +++ b/contrib/pg_tde/documentation/variables.yml @@ -1,9 +1,11 @@ #Variables used throughout the docs -release: 'RC2' +tdeversion: '1.0' +release: '1.0' pgversion17: '17.5' -tdebranch: TDE_REL_17_STABLE +tdebranch: release-17.5.2 date: + GA10: '2025-06-30' RC2: '2025-05-29' RC: '2025-03-27'