From 5df6c586fc433c415ffebdba8adfbbaf5242a759 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Wed, 28 May 2025 17:12:37 +0200 Subject: [PATCH] PG-1257 Add functions for principal key removal Add SQL functions that allow user to remove principal key. * Database level principal key can be removed if there are no encrypted tables or if there is default key. For the first case we just drop key map file completely, for the second we perform key rotation. * Default principal key can be removed if there are no databases that use it. Readded the DELETE key function to docs based on 1257 in Architecture chapter where we also update from DROP to DELETE. --- contrib/pg_tde/Makefile | 9 +- .../documentation/docs/architecture/index.md | 20 ++- contrib/pg_tde/expected/access_control.out | 5 + contrib/pg_tde/expected/alter_index.out | 12 ++ contrib/pg_tde/expected/create_database.out | 12 ++ .../pg_tde/expected/default_principal_key.out | 33 ++-- .../pg_tde/expected/delete_principal_key.out | 130 ++++++++++++++ contrib/pg_tde/expected/key_provider.out | 14 +- contrib/pg_tde/meson.build | 9 +- contrib/pg_tde/pg_tde--1.0-rc.sql | 12 ++ contrib/pg_tde/sql/access_control.sql | 3 + contrib/pg_tde/sql/alter_index.sql | 2 + contrib/pg_tde/sql/create_database.sql | 2 + contrib/pg_tde/sql/default_principal_key.sql | 3 +- contrib/pg_tde/sql/delete_principal_key.sql | 53 ++++++ contrib/pg_tde/src/access/pg_tde_tdemap.c | 41 ++++- contrib/pg_tde/src/access/pg_tde_xlog.c | 14 ++ .../pg_tde/src/catalog/tde_principal_key.c | 169 +++++++++++++++++- .../pg_tde/src/include/access/pg_tde_tdemap.h | 2 + .../pg_tde/src/include/access/pg_tde_xlog.h | 1 + contrib/pg_tde/src/pg_tde_event_capture.c | 6 +- 21 files changed, 506 insertions(+), 46 deletions(-) create mode 100644 contrib/pg_tde/expected/delete_principal_key.out create mode 100644 contrib/pg_tde/sql/delete_principal_key.sql diff --git a/contrib/pg_tde/Makefile b/contrib/pg_tde/Makefile index f8500336c00c2..867b02ee8884a 100644 --- a/contrib/pg_tde/Makefile +++ b/contrib/pg_tde/Makefile @@ -8,12 +8,13 @@ DATA = pg_tde--1.0-rc.sql # Since meson supports skipping test suites this is a make only feature ifndef TDE_MODE REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/pg_tde/pg_tde.conf -# create_database must run after default_principal_key which must run after -# key_provider. REGRESS = access_control \ alter_index \ cache_alloc \ change_access_method \ +create_database \ +default_principal_key \ +delete_principal_key \ insert_update_delete \ key_provider \ kmip_test \ @@ -24,9 +25,7 @@ relocate \ tablespace \ toast_decrypt \ vault_v2_test \ -version \ -default_principal_key \ -create_database +version TAP_TESTS = 1 endif diff --git a/contrib/pg_tde/documentation/docs/architecture/index.md b/contrib/pg_tde/documentation/docs/architecture/index.md index d2061a8f7058c..0eff7081d4b07 100644 --- a/contrib/pg_tde/documentation/docs/architecture/index.md +++ b/contrib/pg_tde/documentation/docs/architecture/index.md @@ -292,15 +292,23 @@ With `pg_tde.inherit_global_key_providers`, it is also possible to set up a defa With this feature, it is possible for the entire database server to easily use the same principal key for all databases, completely disabling multi-tenency. -A default key can be managed with the following functions: +#### Manage a default key -```sql -pg_tde_set_default_key_using_global_key_provider('key-name', 'provider-name', 'true/false') -``` +You can manage a default key with the following functions: + +* `pg_tde_set_default_key_using_global_key_provider('key-name','provider-name','true/false')` +* `pg_tde_delete_default_key()` + +!!! note + `pg_tde_delete_default_key()` is only possible if there's no table currently using the default principal key. + Changing the default principal key will rotate the encryption of internal keys for all databases using the current default principal key. + +#### Delete a key -`DROP` is only possible if there's no table currently using the default principal key. +The `pg_tde_delete_key()` function removes the principal key for the current database. If the current database has any encrypted tables, and there isn’t a default principal key configured, it reports an error instead. If there are encrypted tables, but there’s also a global default principal key, internal keys will be encrypted with the default key. -Changing the default principal key will rotate the encryption of internal keys for all databases using the current default principal key. +!!! note + WAL keys **cannot** be deleted, as server keys are managed separately. ### Current key details diff --git a/contrib/pg_tde/expected/access_control.out b/contrib/pg_tde/expected/access_control.out index 8996168f54fc6..045e3ca8964c3 100644 --- a/contrib/pg_tde/expected/access_control.out +++ b/contrib/pg_tde/expected/access_control.out @@ -10,6 +10,8 @@ SET ROLE regress_pg_tde_access_control; -- should throw access denied SELECT pg_tde_set_key_using_database_key_provider('test-db-key', 'local-file-provider'); ERROR: permission denied for function pg_tde_set_key_using_database_key_provider +SELECT pg_tde_delete_key(); +ERROR: permission denied for function pg_tde_delete_key SELECT pg_tde_list_all_database_key_providers(); ERROR: permission denied for function pg_tde_list_all_database_key_providers SELECT pg_tde_list_all_global_key_providers(); @@ -37,6 +39,7 @@ GRANT EXECUTE ON FUNCTION pg_tde_delete_global_key_provider(TEXT) TO regress_pg_ GRANT EXECUTE ON FUNCTION pg_tde_set_default_key_using_global_key_provider(TEXT, TEXT, BOOLEAN) TO regress_pg_tde_access_control; GRANT EXECUTE ON FUNCTION pg_tde_set_key_using_global_key_provider(TEXT, TEXT, BOOLEAN) TO regress_pg_tde_access_control; GRANT EXECUTE ON FUNCTION pg_tde_set_server_key_using_global_key_provider(TEXT, TEXT, BOOLEAN) TO regress_pg_tde_access_control; +GRANT EXECUTE ON FUNCTION pg_tde_delete_default_key() TO regress_pg_tde_access_control; SET ROLE regress_pg_tde_access_control; SELECT pg_tde_add_database_key_provider_file('local-file-provider', '/tmp/pg_tde_test_keyring.per'); ERROR: must be superuser to modify key providers @@ -56,5 +59,7 @@ SELECT pg_tde_set_default_key_using_global_key_provider('key1', 'global-file-pro ERROR: must be superuser to access global key providers SELECT pg_tde_set_server_key_using_global_key_provider('key1', 'global-file-provider'); ERROR: must be superuser to access global key providers +SELECT pg_tde_delete_default_key(); +ERROR: must be superuser to access global key providers RESET ROLE; DROP EXTENSION pg_tde CASCADE; diff --git a/contrib/pg_tde/expected/alter_index.out b/contrib/pg_tde/expected/alter_index.out index 72575e8a548e6..dc3c181acdd49 100644 --- a/contrib/pg_tde/expected/alter_index.out +++ b/contrib/pg_tde/expected/alter_index.out @@ -67,5 +67,17 @@ SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_ind (5 rows) DROP TABLE concur_reindex_part; +SELECT pg_tde_delete_key(); + pg_tde_delete_key +------------------- + +(1 row) + +SELECT pg_tde_delete_database_key_provider('file-vault'); + pg_tde_delete_database_key_provider +------------------------------------- + +(1 row) + DROP EXTENSION pg_tde; RESET default_table_access_method; diff --git a/contrib/pg_tde/expected/create_database.out b/contrib/pg_tde/expected/create_database.out index 16acc9d8d4c2f..83944edd3e3e5 100644 --- a/contrib/pg_tde/expected/create_database.out +++ b/contrib/pg_tde/expected/create_database.out @@ -102,4 +102,16 @@ CREATE DATABASE new_db_file_copy TEMPLATE template_db STRATEGY FILE_COPY; DROP DATABASE new_db_file_copy; DROP DATABASE new_db; DROP DATABASE template_db; +SELECT pg_tde_delete_default_key(); + pg_tde_delete_default_key +--------------------------- + +(1 row) + +SELECT pg_tde_delete_global_key_provider('global-file-vault'); + pg_tde_delete_global_key_provider +----------------------------------- + +(1 row) + DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/default_principal_key.out b/contrib/pg_tde/expected/default_principal_key.out index b36393ff0aa51..ad5870cc198e0 100644 --- a/contrib/pg_tde/expected/default_principal_key.out +++ b/contrib/pg_tde/expected/default_principal_key.out @@ -33,20 +33,17 @@ SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_default_key_info(); key_provider_id | key_provider_name | key_name -----------------+-------------------+------------- - -5 | file-provider | default-key + -2 | file-provider | default-key (1 row) -- fails SELECT pg_tde_delete_global_key_provider('file-provider'); ERROR: Can't delete a provider which is currently in use SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); - id | provider_name -----+------------------ - -1 | file-keyring - -3 | global-provider - -4 | global-provider2 - -5 | file-provider -(4 rows) + id | provider_name +----+--------------- + -2 | file-provider +(1 row) -- Should fail: no principal key for the database yet SELECT key_provider_id, key_provider_name, key_name @@ -68,7 +65,7 @@ SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); key_provider_id | key_provider_name | key_name -----------------+-------------------+------------- - -5 | file-provider | default-key + -2 | file-provider | default-key (1 row) SELECT current_database() AS regress_database @@ -97,7 +94,7 @@ SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); key_provider_id | key_provider_name | key_name -----------------+-------------------+------------- - -5 | file-provider | default-key + -2 | file-provider | default-key (1 row) \c :regress_database @@ -112,7 +109,7 @@ SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); key_provider_id | key_provider_name | key_name -----------------+-------------------+----------------- - -5 | file-provider | new-default-key + -2 | file-provider | new-default-key (1 row) \c regress_pg_tde_other @@ -120,7 +117,7 @@ SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); key_provider_id | key_provider_name | key_name -----------------+-------------------+----------------- - -5 | file-provider | new-default-key + -2 | file-provider | new-default-key (1 row) SELECT pg_buffercache_evict(bufferid) FROM pg_buffercache WHERE relfilenode = (SELECT relfilenode FROM pg_class WHERE oid = 'test_enc'::regclass); @@ -155,6 +152,18 @@ SELECT * FROM test_enc; (3 rows) DROP TABLE test_enc; +SELECT pg_tde_delete_default_key(); + pg_tde_delete_default_key +--------------------------- + +(1 row) + +SELECT pg_tde_delete_global_key_provider('file-provider'); + pg_tde_delete_global_key_provider +----------------------------------- + +(1 row) + DROP EXTENSION pg_tde CASCADE; DROP EXTENSION pg_buffercache; DROP DATABASE regress_pg_tde_other; diff --git a/contrib/pg_tde/expected/delete_principal_key.out b/contrib/pg_tde/expected/delete_principal_key.out new file mode 100644 index 0000000000000..480297556dd07 --- /dev/null +++ b/contrib/pg_tde/expected/delete_principal_key.out @@ -0,0 +1,130 @@ +CREATE EXTENSION IF NOT EXISTS pg_tde; +SELECT pg_tde_add_global_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); + pg_tde_add_global_key_provider_file +------------------------------------- + +(1 row) + +-- Set the local key and delete it without any encrypted tables +-- Should succeed: nothing used the key +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); + pg_tde_set_key_using_global_key_provider +------------------------------------------ + +(1 row) + +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+------------- + -3 | file-provider | test-db-key +(1 row) + +SELECT pg_tde_delete_key(); + pg_tde_delete_key +------------------- + +(1 row) + +-- Set local key, encrypt a table, and delete the key +-- Should fail: the is no default key to fallback +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); + pg_tde_set_key_using_global_key_provider +------------------------------------------ + +(1 row) + +CREATE TABLE test_table (id int, data text) USING tde_heap; +SELECT pg_tde_delete_key(); +ERROR: cannot delete principal key +DETAIL: There are encrypted tables in the database. +HINT: Set default principal key as fallback option or decrypt all tables before deleting principal key. +-- Decrypt the table and delete the key +-- Should succeed: there is no more encrypted tables +ALTER TABLE test_table SET ACCESS METHOD heap; +SELECT pg_tde_delete_key(); + pg_tde_delete_key +------------------- + +(1 row) + +-- Set local key, encrypt the table then delete teable and key +-- Should succeed: the table is deleted and there are no more encrypted tables +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); + pg_tde_set_key_using_global_key_provider +------------------------------------------ + +(1 row) + +ALTER TABLE test_table SET ACCESS METHOD tde_heap; +DROP TABLE test_table; +SELECT pg_tde_delete_key(); + pg_tde_delete_key +------------------- + +(1 row) + +-- Set default key, set regular key, create table, delete regular key +-- Should succeed: regular key will be rotated to default key +SELECT pg_tde_set_default_key_using_global_key_provider('defalut-key','file-provider'); + pg_tde_set_default_key_using_global_key_provider +-------------------------------------------------- + +(1 row) + +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); + pg_tde_set_key_using_global_key_provider +------------------------------------------ + +(1 row) + +CREATE TABLE test_table (id int, data text) USING tde_heap; +SELECT pg_tde_delete_key(); + pg_tde_delete_key +------------------- + +(1 row) + +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + key_provider_id | key_provider_name | key_name +-----------------+-------------------+------------- + -3 | file-provider | defalut-key +(1 row) + +-- Try to delete key when default key is used +-- Should fail: table already uses the default key, so there is no key to fallback to +SELECT pg_tde_delete_key(); +ERROR: cannot delete principal key +DETAIL: There are encrypted tables in the database. +-- Try to delete default key +-- Should fail: default key is used by the table +SELECT pg_tde_delete_default_key(); +ERROR: cannot delete default principal key +HINT: There are encrypted tables in the database with id: 16384. +-- Set regular principal key, delete default key +-- Should succeed: the table will use the regular key +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); + pg_tde_set_key_using_global_key_provider +------------------------------------------ + +(1 row) + +SELECT pg_tde_delete_default_key(); + pg_tde_delete_default_key +--------------------------- + +(1 row) + +DROP TABLE test_table; +SELECT pg_tde_delete_key(); + pg_tde_delete_key +------------------- + +(1 row) + +SELECT pg_tde_delete_global_key_provider('file-provider'); + pg_tde_delete_global_key_provider +----------------------------------- + +(1 row) + +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/expected/key_provider.out b/contrib/pg_tde/expected/key_provider.out index e3ef0dd2e4aa1..c77c2653657d1 100644 --- a/contrib/pg_tde/expected/key_provider.out +++ b/contrib/pg_tde/expected/key_provider.out @@ -75,8 +75,8 @@ SELECT pg_tde_add_global_key_provider_file('file-keyring2','/tmp/pg_tde_test_key SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- - -1 | file-keyring - -2 | file-keyring2 + -4 | file-keyring + -5 | file-keyring2 (2 rows) -- fails @@ -105,8 +105,8 @@ SELECT id, provider_name FROM pg_tde_list_all_database_key_providers(); SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- - -1 | file-keyring - -2 | file-keyring2 + -4 | file-keyring + -5 | file-keyring2 (2 rows) SELECT pg_tde_set_key_using_global_key_provider('test-db-key', 'file-keyring', false); @@ -121,8 +121,8 @@ ERROR: Can't delete a provider which is currently in use SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- - -1 | file-keyring - -2 | file-keyring2 + -4 | file-keyring + -5 | file-keyring2 (2 rows) -- works @@ -135,7 +135,7 @@ SELECT pg_tde_delete_global_key_provider('file-keyring2'); SELECT id, provider_name FROM pg_tde_list_all_global_key_providers(); id | provider_name ----+--------------- - -1 | file-keyring + -4 | file-keyring (1 row) -- Creating a file key provider fails if we can't open or create the file diff --git a/contrib/pg_tde/meson.build b/contrib/pg_tde/meson.build index 7abf96de07030..a6e5a52a7e1d7 100644 --- a/contrib/pg_tde/meson.build +++ b/contrib/pg_tde/meson.build @@ -80,26 +80,25 @@ install_data( kwargs: contrib_data_args, ) -# create_database must run after default_principal_key which must run after -# key_provider. sql_tests = [ 'access_control', 'alter_index', 'cache_alloc', 'change_access_method', + 'create_database', + 'default_principal_key', + 'delete_principal_key', 'insert_update_delete', 'key_provider', 'kmip_test', 'partition_table', 'pg_tde_is_encrypted', - 'relocate', 'recreate_storage', + 'relocate', 'tablespace', 'toast_decrypt', 'vault_v2_test', 'version', - 'default_principal_key', - 'create_database', ] tap_tests = [ diff --git a/contrib/pg_tde/pg_tde--1.0-rc.sql b/contrib/pg_tde/pg_tde--1.0-rc.sql index cc92bbe217b92..c9092e81ee66e 100644 --- a/contrib/pg_tde/pg_tde--1.0-rc.sql +++ b/contrib/pg_tde/pg_tde--1.0-rc.sql @@ -259,6 +259,18 @@ LANGUAGE C AS 'MODULE_PATHNAME'; REVOKE ALL ON FUNCTION pg_tde_verify_default_key() FROM PUBLIC; +CREATE FUNCTION pg_tde_delete_key() +RETURNS VOID +LANGUAGE C +AS 'MODULE_PATHNAME'; +REVOKE ALL ON FUNCTION pg_tde_delete_key() FROM PUBLIC; + +CREATE FUNCTION pg_tde_delete_default_key() +RETURNS VOID +LANGUAGE C +AS 'MODULE_PATHNAME'; +REVOKE ALL ON FUNCTION pg_tde_delete_default_key() FROM PUBLIC; + CREATE FUNCTION pg_tde_key_info() RETURNS TABLE ( key_name TEXT, key_provider_name TEXT, diff --git a/contrib/pg_tde/sql/access_control.sql b/contrib/pg_tde/sql/access_control.sql index 90ca5e9c60bc7..b8ac7aff0ec79 100644 --- a/contrib/pg_tde/sql/access_control.sql +++ b/contrib/pg_tde/sql/access_control.sql @@ -8,6 +8,7 @@ SET ROLE regress_pg_tde_access_control; -- should throw access denied SELECT pg_tde_set_key_using_database_key_provider('test-db-key', 'local-file-provider'); +SELECT pg_tde_delete_key(); SELECT pg_tde_list_all_database_key_providers(); SELECT pg_tde_list_all_global_key_providers(); SELECT pg_tde_key_info(); @@ -29,6 +30,7 @@ GRANT EXECUTE ON FUNCTION pg_tde_delete_global_key_provider(TEXT) TO regress_pg_ GRANT EXECUTE ON FUNCTION pg_tde_set_default_key_using_global_key_provider(TEXT, TEXT, BOOLEAN) TO regress_pg_tde_access_control; GRANT EXECUTE ON FUNCTION pg_tde_set_key_using_global_key_provider(TEXT, TEXT, BOOLEAN) TO regress_pg_tde_access_control; GRANT EXECUTE ON FUNCTION pg_tde_set_server_key_using_global_key_provider(TEXT, TEXT, BOOLEAN) TO regress_pg_tde_access_control; +GRANT EXECUTE ON FUNCTION pg_tde_delete_default_key() TO regress_pg_tde_access_control; SET ROLE regress_pg_tde_access_control; @@ -41,6 +43,7 @@ SELECT pg_tde_delete_global_key_provider('global-file-provider'); SELECT pg_tde_set_key_using_global_key_provider('key1', 'global-file-provider'); SELECT pg_tde_set_default_key_using_global_key_provider('key1', 'global-file-provider'); SELECT pg_tde_set_server_key_using_global_key_provider('key1', 'global-file-provider'); +SELECT pg_tde_delete_default_key(); RESET ROLE; diff --git a/contrib/pg_tde/sql/alter_index.sql b/contrib/pg_tde/sql/alter_index.sql index 9dac7bea58338..794161bbd0eae 100644 --- a/contrib/pg_tde/sql/alter_index.sql +++ b/contrib/pg_tde/sql/alter_index.sql @@ -33,5 +33,7 @@ SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_ind SELECT relid, parentrelid, level FROM pg_partition_tree('concur_reindex_part_index') ORDER BY relid, level; DROP TABLE concur_reindex_part; +SELECT pg_tde_delete_key(); +SELECT pg_tde_delete_database_key_provider('file-vault'); DROP EXTENSION pg_tde; RESET default_table_access_method; diff --git a/contrib/pg_tde/sql/create_database.sql b/contrib/pg_tde/sql/create_database.sql index d62cdc12f5092..77c7aaf84a83a 100644 --- a/contrib/pg_tde/sql/create_database.sql +++ b/contrib/pg_tde/sql/create_database.sql @@ -59,4 +59,6 @@ DROP DATABASE new_db_file_copy; DROP DATABASE new_db; DROP DATABASE template_db; +SELECT pg_tde_delete_default_key(); +SELECT pg_tde_delete_global_key_provider('global-file-vault'); DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/default_principal_key.sql b/contrib/pg_tde/sql/default_principal_key.sql index be4183566c7fa..b91744390daa5 100644 --- a/contrib/pg_tde/sql/default_principal_key.sql +++ b/contrib/pg_tde/sql/default_principal_key.sql @@ -93,7 +93,8 @@ SELECT pg_buffercache_evict(bufferid) FROM pg_buffercache WHERE relfilenode = (S SELECT * FROM test_enc; DROP TABLE test_enc; - +SELECT pg_tde_delete_default_key(); +SELECT pg_tde_delete_global_key_provider('file-provider'); DROP EXTENSION pg_tde CASCADE; DROP EXTENSION pg_buffercache; diff --git a/contrib/pg_tde/sql/delete_principal_key.sql b/contrib/pg_tde/sql/delete_principal_key.sql new file mode 100644 index 0000000000000..6f313277ab297 --- /dev/null +++ b/contrib/pg_tde/sql/delete_principal_key.sql @@ -0,0 +1,53 @@ +CREATE EXTENSION IF NOT EXISTS pg_tde; + +SELECT pg_tde_add_global_key_provider_file('file-provider','/tmp/pg_tde_test_keyring.per'); + +-- Set the local key and delete it without any encrypted tables +-- Should succeed: nothing used the key +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); +SELECT pg_tde_delete_key(); + +-- Set local key, encrypt a table, and delete the key +-- Should fail: the is no default key to fallback +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); +CREATE TABLE test_table (id int, data text) USING tde_heap; +SELECT pg_tde_delete_key(); + +-- Decrypt the table and delete the key +-- Should succeed: there is no more encrypted tables +ALTER TABLE test_table SET ACCESS METHOD heap; +SELECT pg_tde_delete_key(); + +-- Set local key, encrypt the table then delete teable and key +-- Should succeed: the table is deleted and there are no more encrypted tables +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); +ALTER TABLE test_table SET ACCESS METHOD tde_heap; +DROP TABLE test_table; +SELECT pg_tde_delete_key(); + +-- Set default key, set regular key, create table, delete regular key +-- Should succeed: regular key will be rotated to default key +SELECT pg_tde_set_default_key_using_global_key_provider('defalut-key','file-provider'); +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); +CREATE TABLE test_table (id int, data text) USING tde_heap; +SELECT pg_tde_delete_key(); +SELECT key_provider_id, key_provider_name, key_name FROM pg_tde_key_info(); + +-- Try to delete key when default key is used +-- Should fail: table already uses the default key, so there is no key to fallback to +SELECT pg_tde_delete_key(); + +-- Try to delete default key +-- Should fail: default key is used by the table +SELECT pg_tde_delete_default_key(); + +-- Set regular principal key, delete default key +-- Should succeed: the table will use the regular key +SELECT pg_tde_set_key_using_global_key_provider('test-db-key','file-provider'); +SELECT pg_tde_delete_default_key(); + +DROP TABLE test_table; +SELECT pg_tde_delete_key(); +SELECT pg_tde_delete_global_key_provider('file-provider'); +DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index 3ab258ea05ec3..d4e377559eefd 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -508,6 +508,40 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p } } +void +pg_tde_delete_principal_key_redo(Oid dbOid) +{ + char path[MAXPGPATH]; + + pg_tde_set_db_file_path(dbOid, path); + + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + durable_unlink(path, WARNING); + LWLockRelease(tde_lwlock_enc_keys()); +} + +/* + * Deletes the principal key for the database. This fucntion checks if key map + * file has any entries, and if not, it removes the file. Otherwise raises an error. + */ +void +pg_tde_delete_principal_key(Oid dbOid) +{ + char path[MAXPGPATH]; + + Assert(LWLockHeldByMeInMode(tde_lwlock_enc_keys(), LW_EXCLUSIVE)); + Assert(pg_tde_count_relations(dbOid) == 0); + + pg_tde_set_db_file_path(dbOid, path); + + XLogBeginInsert(); + XLogRegisterData((char *) &dbOid, sizeof(Oid)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_DELETE_PRINCIPAL_KEY); + + /* Remove whole key map file */ + durable_unlink(path, ERROR); +} + /* * It's called by seg_write inside crit section so no pallocs, hence * needs keyfile_path @@ -652,15 +686,14 @@ int pg_tde_count_relations(Oid dbOid) { char db_map_path[MAXPGPATH]; - LWLock *lock_pk = tde_lwlock_enc_keys(); File map_fd; off_t curr_pos = 0; TDEMapEntry map_entry; int count = 0; - pg_tde_set_db_file_path(dbOid, db_map_path); + Assert(LWLockHeldByMeInMode(tde_lwlock_enc_keys(), LW_SHARED) || LWLockHeldByMeInMode(tde_lwlock_enc_keys(), LW_EXCLUSIVE)); - LWLockAcquire(lock_pk, LW_SHARED); + pg_tde_set_db_file_path(dbOid, db_map_path); map_fd = pg_tde_open_file_read(db_map_path, true, &curr_pos); if (map_fd < 0) @@ -674,8 +707,6 @@ pg_tde_count_relations(Oid dbOid) CloseTransientFile(map_fd); - LWLockRelease(lock_pk); - return count; } diff --git a/contrib/pg_tde/src/access/pg_tde_xlog.c b/contrib/pg_tde/src/access/pg_tde_xlog.c index 14015f32b8d1e..c0e1ba07510d4 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog.c @@ -73,6 +73,12 @@ tdeheap_rmgr_redo(XLogReaderState *record) xl_tde_perform_rotate_key(xlrec); } + else if (info == XLOG_TDE_DELETE_PRINCIPAL_KEY) + { + Oid dbOid = *((Oid *) XLogRecGetData(record)); + + pg_tde_delete_principal_key_redo(dbOid); + } else if (info == XLOG_TDE_WRITE_KEY_PROVIDER) { KeyringProviderRecordInFile *xlrec = (KeyringProviderRecordInFile *) XLogRecGetData(record); @@ -114,6 +120,12 @@ tdeheap_rmgr_desc(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "db: %u", xlrec->databaseId); } + else if (info == XLOG_TDE_DELETE_PRINCIPAL_KEY) + { + Oid dbOid = *((Oid *) XLogRecGetData(record)); + + appendStringInfo(buf, "db: %u", dbOid); + } else if (info == XLOG_TDE_WRITE_KEY_PROVIDER) { KeyringProviderRecordInFile *xlrec = (KeyringProviderRecordInFile *) XLogRecGetData(record); @@ -139,6 +151,8 @@ tdeheap_rmgr_identify(uint8 info) return "ADD_PRINCIPAL_KEY"; case XLOG_TDE_ROTATE_PRINCIPAL_KEY: return "ROTATE_PRINCIPAL_KEY"; + case XLOG_TDE_DELETE_PRINCIPAL_KEY: + return "DELETE_PRINCIPAL_KEY"; case XLOG_TDE_WRITE_KEY_PROVIDER: return "WRITE_KEY_PROVIDER"; case XLOG_TDE_INSTALL_EXTENSION: diff --git a/contrib/pg_tde/src/catalog/tde_principal_key.c b/contrib/pg_tde/src/catalog/tde_principal_key.c index b168bb578f41a..71bb95b69c0e5 100644 --- a/contrib/pg_tde/src/catalog/tde_principal_key.c +++ b/contrib/pg_tde/src/catalog/tde_principal_key.c @@ -39,12 +39,14 @@ #ifndef FRONTEND #include "access/genam.h" +#include "access/heapam.h" #include "access/table.h" +#include "access/tableam.h" +#include "common/pg_tde_shmem.h" #include "funcapi.h" #include "lib/dshash.h" #include "storage/lwlock.h" #include "storage/shmem.h" -#include "common/pg_tde_shmem.h" #else #include "pg_tde_fe.h" #endif @@ -92,7 +94,7 @@ static void clear_principal_key_cache(Oid databaseId); static inline dshash_table *get_principal_key_hash(void); static TDEPrincipalKey *get_principal_key_from_cache(Oid dbOid); static bool pg_tde_is_same_principal_key(TDEPrincipalKey *a, TDEPrincipalKey *b); -static void pg_tde_update_global_principal_key_everywhere(TDEPrincipalKey *oldKey, TDEPrincipalKey *newKey); +static void pg_tde_update_default_principal_key_everywhere(TDEPrincipalKey *oldKey, TDEPrincipalKey *newKey); static void push_principal_key_to_cache(TDEPrincipalKey *principalKey); static Datum pg_tde_get_key_info(PG_FUNCTION_ARGS, Oid dbOid); static TDEPrincipalKey *get_principal_key_from_keyring(Oid dbOid); @@ -103,11 +105,14 @@ static void set_principal_key_with_keyring(const char *key_name, Oid dbOid, bool ensure_new_key); static bool pg_tde_verify_principal_key_internal(Oid databaseOid); +static void pg_tde_rotate_default_key_for_database(TDEPrincipalKey *oldKey, TDEPrincipalKey *newKeyTemplate); PG_FUNCTION_INFO_V1(pg_tde_set_default_key_using_global_key_provider); PG_FUNCTION_INFO_V1(pg_tde_set_key_using_database_key_provider); PG_FUNCTION_INFO_V1(pg_tde_set_key_using_global_key_provider); PG_FUNCTION_INFO_V1(pg_tde_set_server_key_using_global_key_provider); +PG_FUNCTION_INFO_V1(pg_tde_delete_key); +PG_FUNCTION_INFO_V1(pg_tde_delete_default_key); static void pg_tde_set_principal_key_internal(Oid providerOid, Oid dbOid, const char *principal_key_name, const char *provider_name, bool ensure_new_key); @@ -573,10 +578,157 @@ pg_tde_set_principal_key_internal(Oid providerOid, Oid dbOid, const char *key_na LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); newDefaultKey = GetPrincipalKeyNoDefault(dbOid, LW_EXCLUSIVE); - pg_tde_update_global_principal_key_everywhere(&existingKeyCopy, newDefaultKey); + pg_tde_update_default_principal_key_everywhere(&existingKeyCopy, newDefaultKey); + + LWLockRelease(tde_lwlock_enc_keys()); + } +} + + +/* + * SQL interface to delete principal key. + * + * This operation allowed if there is no any encrypted tables in the database or + * if the default principal key is set for the database. In second case, + * key for database rotated to the default key. + */ +Datum +pg_tde_delete_key(PG_FUNCTION_ARGS) +{ + TDEPrincipalKey *principal_key; + TDEPrincipalKey *default_principal_key; + + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + + principal_key = GetPrincipalKeyNoDefault(MyDatabaseId, LW_EXCLUSIVE); + if (principal_key == NULL) + ereport(ERROR, errmsg("principal key does not exists for the database")); + + ereport(LOG, errmsg("Deleting principal key [%s] for the database", principal_key->keyInfo.name)); + + /* + * If database has something encryted, we can try to fallback to the + * default principal key + */ + if (pg_tde_count_relations(MyDatabaseId) != 0) + { + default_principal_key = GetPrincipalKeyNoDefault(DEFAULT_DATA_TDE_OID, LW_EXCLUSIVE); + if (default_principal_key == NULL) + { + ereport(ERROR, + errmsg("cannot delete principal key"), + errdetail("There are encrypted tables in the database."), + errhint("Set default principal key as fallback option or decrypt all tables before deleting principal key.")); + } + + /* + * If database already encrypted with default principal key, there is + * nothing to do + */ + if (pg_tde_is_same_principal_key(principal_key, default_principal_key)) + { + ereport(ERROR, + errmsg("cannot delete principal key"), + errdetail("There are encrypted tables in the database.")); + } + + pg_tde_rotate_default_key_for_database(principal_key, default_principal_key); LWLockRelease(tde_lwlock_enc_keys()); + PG_RETURN_VOID(); } + + pg_tde_delete_principal_key(MyDatabaseId); + clear_principal_key_cache(MyDatabaseId); + + LWLockRelease(tde_lwlock_enc_keys()); + PG_RETURN_VOID(); +} + +/* + * SQL interface to delete default principal key. + * + * This operation allowed if there is no databases using the default principal key. + */ +Datum +pg_tde_delete_default_key(PG_FUNCTION_ARGS) +{ + HeapTuple tuple; + SysScanDesc scan; + Relation rel; + TDEPrincipalKey *principal_key; + TDEPrincipalKey *default_principal_key; + List *dbs = NIL; + + if (!superuser()) + ereport(ERROR, + errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("must be superuser to access global key providers")); + + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + + default_principal_key = GetPrincipalKeyNoDefault(DEFAULT_DATA_TDE_OID, LW_EXCLUSIVE); + if (default_principal_key == NULL) + ereport(ERROR, errmsg("default principal key is not set")); + + ereport(LOG, errmsg("Deleting default principal key [%s]", default_principal_key->keyInfo.name)); + + /* + * Take row exclusive lock, as we do not want anybody to create/drop a + * database in parallel. If it happens, its not the end of the world, but + * not ideal. + */ + rel = table_open(DatabaseRelationId, RowExclusiveLock); + scan = systable_beginscan(rel, 0, false, NULL, 0, NULL); + + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Oid dbOid = ((Form_pg_database) GETSTRUCT(tuple))->oid; + + principal_key = GetPrincipalKeyNoDefault(dbOid, LW_EXCLUSIVE); + + /* Check if database uses default principalkey */ + if (pg_tde_is_same_principal_key(default_principal_key, principal_key)) + { + /* + * If database key map is non-empty raise an error, as we cannot + * delete default principal key if there are encrypted tables in + * the database. + */ + if (pg_tde_count_relations(dbOid) != 0) + { + ereport(ERROR, + errmsg("cannot delete default principal key"), + errhint("There are encrypted tables in the database with id: %u.", dbOid)); + } + + /* Remember databases that has no encrypted tables */ + dbs = lappend_oid(dbs, dbOid); + } + } + + /* + * Remove empty key map files for databases that has no encrypted tables + * as we cannot leave reference to the default principal key. + */ + foreach_oid(dbOid, dbs) + { + pg_tde_delete_principal_key(dbOid); + clear_principal_key_cache(dbOid); + } + + systable_endscan(scan); + table_close(rel, RowExclusiveLock); + + /* No databases use default principal key, so we can delete it */ + pg_tde_delete_principal_key(DEFAULT_DATA_TDE_OID); + clear_principal_key_cache(DEFAULT_DATA_TDE_OID); + + LWLockRelease(tde_lwlock_enc_keys()); + + list_free(dbs); + + PG_RETURN_VOID(); } PG_FUNCTION_INFO_V1(pg_tde_key_info); @@ -1028,8 +1180,17 @@ pg_tde_rotate_default_key_for_database(TDEPrincipalKey *oldKey, TDEPrincipalKey pfree(newKey); } +/* + * Update the default principal key for all databases that use it. + * + * This function is called when the default principal key is rotated. It + * updates all databases that use the old default principal key to use the new + * one. + * + * Caller should hold an exclusive tde_lwlock_enc_keys lock. + */ static void -pg_tde_update_global_principal_key_everywhere(TDEPrincipalKey *oldKey, TDEPrincipalKey *newKey) +pg_tde_update_default_principal_key_everywhere(TDEPrincipalKey *oldKey, TDEPrincipalKey *newKey) { HeapTuple tuple; SysScanDesc scan; diff --git a/contrib/pg_tde/src/include/access/pg_tde_tdemap.h b/contrib/pg_tde/src/include/access/pg_tde_tdemap.h index eed5b7bc569d0..0ca77332a96f6 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_tdemap.h +++ b/contrib/pg_tde/src/include/access/pg_tde_tdemap.h @@ -105,6 +105,8 @@ extern bool pg_tde_verify_principal_key_info(TDESignedPrincipalKeyInfo *signed_k extern void pg_tde_save_principal_key(const TDEPrincipalKey *principal_key, bool write_xlog); extern void pg_tde_save_principal_key_redo(const TDESignedPrincipalKeyInfo *signed_key_info); extern void pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key, bool write_xlog); +extern void pg_tde_delete_principal_key(Oid dbOid); +extern void pg_tde_delete_principal_key_redo(Oid dbOid); const char *tde_sprint_key(InternalKey *k); diff --git a/contrib/pg_tde/src/include/access/pg_tde_xlog.h b/contrib/pg_tde/src/include/access/pg_tde_xlog.h index 2f08fecb37162..5cf5e9c78b7d9 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_xlog.h +++ b/contrib/pg_tde/src/include/access/pg_tde_xlog.h @@ -18,6 +18,7 @@ #define XLOG_TDE_WRITE_KEY_PROVIDER 0x30 #define XLOG_TDE_INSTALL_EXTENSION 0x40 #define XLOG_TDE_REMOVE_RELATION_KEY 0x50 +#define XLOG_TDE_DELETE_PRINCIPAL_KEY 0x60 /* ID 140 is registered for Percona TDE extension: https://wiki.postgresql.org/wiki/CustomWALResourceManagers */ #define RM_TDERMGR_ID 140 diff --git a/contrib/pg_tde/src/pg_tde_event_capture.c b/contrib/pg_tde/src/pg_tde_event_capture.c index 68466d5905fc4..5124f209781a2 100644 --- a/contrib/pg_tde/src/pg_tde_event_capture.c +++ b/contrib/pg_tde/src/pg_tde_event_capture.c @@ -635,7 +635,11 @@ pg_tde_proccess_utility(PlannedStmt *pstmt, if (dbOid != InvalidOid) { - int count = pg_tde_count_relations(dbOid); + int count; + + LWLockAcquire(tde_lwlock_enc_keys(), LW_SHARED); + count = pg_tde_count_relations(dbOid); + LWLockRelease(tde_lwlock_enc_keys()); if (count > 0) ereport(ERROR,