From 89f4ada098d492667cad92c378021cb8d52e2889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85strand?= Date: Thu, 5 Jun 2025 11:05:46 +0200 Subject: [PATCH 1/3] Make tde_mdcreate more strict in it's behavior Only create keys when MAIN fork is created, and trust tde_smgr_should_encrypt() to know when to encrypt. Also trust that the key has already been created if we're in recovery or replication. --- contrib/pg_tde/src/smgr/pg_tde_smgr.c | 49 ++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/contrib/pg_tde/src/smgr/pg_tde_smgr.c b/contrib/pg_tde/src/smgr/pg_tde_smgr.c index cf3de6cd5bcef..bed0d8e5fb61f 100644 --- a/contrib/pg_tde/src/smgr/pg_tde_smgr.c +++ b/contrib/pg_tde/src/smgr/pg_tde_smgr.c @@ -322,6 +322,7 @@ static void tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool isRedo) { TDESMgrRelation *tdereln = (TDESMgrRelation *) reln; + InternalKey *key; /* Copied from mdcreate() in md.c */ if (isRedo && tdereln->md_num_open_segs[forknum] > 0) @@ -334,36 +335,46 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool mdcreate(relold, reln, forknum, isRedo); - if (forknum == MAIN_FORKNUM || forknum == INIT_FORKNUM) + if (forknum != MAIN_FORKNUM) { /* - * Only create keys when creating the main/init fork. Other forks can - * be created later, even during tde creation events. We definitely do + * Only create keys when creating the main fork. Other forks can be + * created later, even during tde creation events. We definitely do * not want to create keys then, even later, when we encrypt all * forks! * * Later calls then decide to encrypt or not based on the existence of * the key. - * - * Since event triggers do not fire on the standby or in recovery we - * do not try to generate any new keys and instead trust the xlog. */ - InternalKey *key = tde_smgr_get_key(&reln->smgr_rlocator); + return; + } - if (!isRedo && !key && tde_smgr_should_encrypt(&reln->smgr_rlocator, &relold)) - key = tde_smgr_create_key(&reln->smgr_rlocator); + if (!tde_smgr_should_encrypt(&reln->smgr_rlocator, &relold)) + { + tdereln->encryption_status = RELATION_NOT_ENCRYPTED; + return; + } - if (key) - { - tdereln->encryption_status = RELATION_KEY_AVAILABLE; - tdereln->relKey = *key; - pfree(key); - } - else - { - tdereln->encryption_status = RELATION_NOT_ENCRYPTED; - } + if (isRedo) + { + /* + * If we're in redo, the WAL record for creating the key has already + * happened and we can just fetch it. + */ + key = tde_smgr_get_key(&reln->smgr_rlocator); + + Assert(key); + if (!key) + elog(ERROR, "could not get key when creating encrypted relation"); } + else + { + key = tde_smgr_create_key(&reln->smgr_rlocator); + } + + tdereln->encryption_status = RELATION_KEY_AVAILABLE; + tdereln->relKey = *key; + pfree(key); } /* From 36766f59cecb072f4ffb6ad7b4e7c28cff4b3848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85strand?= Date: Mon, 9 Jun 2025 11:58:38 +0200 Subject: [PATCH 2/3] Move WAL logging of relation key creation to smgr Instead of it being done down in tdemap code controlled with a boolean, we just do it where the decision to do it is made instead. --- contrib/pg_tde/src/access/pg_tde_tdemap.c | 16 +--------------- .../pg_tde/src/include/access/pg_tde_tdemap.h | 2 +- contrib/pg_tde/src/smgr/pg_tde_smgr.c | 19 +++++++++++++++++-- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/contrib/pg_tde/src/access/pg_tde_tdemap.c b/contrib/pg_tde/src/access/pg_tde_tdemap.c index 0137b9e95ddc7..eb145194288a5 100644 --- a/contrib/pg_tde/src/access/pg_tde_tdemap.c +++ b/contrib/pg_tde/src/access/pg_tde_tdemap.c @@ -76,13 +76,10 @@ static void finalize_key_rotation(const char *path_old, const char *path_new); static int pg_tde_open_file_write(const char *tde_filename, const TDESignedPrincipalKeyInfo *signed_key_info, bool truncate, off_t *curr_pos); void -pg_tde_save_smgr_key(RelFileLocator rel, const InternalKey *rel_key_data, bool write_xlog) +pg_tde_save_smgr_key(RelFileLocator rel, const InternalKey *rel_key_data) { TDEPrincipalKey *principal_key; LWLock *lock_pk = tde_lwlock_enc_keys(); - XLogRelKey xlrec = { - .rlocator = rel, - }; LWLockAcquire(lock_pk, LW_EXCLUSIVE); principal_key = GetPrincipalKey(rel.dbOid, LW_EXCLUSIVE); @@ -95,17 +92,6 @@ pg_tde_save_smgr_key(RelFileLocator rel, const InternalKey *rel_key_data, bool w pg_tde_write_key_map_entry(&rel, rel_key_data, principal_key); LWLockRelease(lock_pk); - - if (write_xlog) - { - /* - * It is fine to write the to WAL after writing to the file since we - * have not WAL logged the SMGR CREATE event either. - */ - XLogBeginInsert(); - XLogRegisterData((char *) &xlrec, sizeof(xlrec)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY); - } } const char * diff --git a/contrib/pg_tde/src/include/access/pg_tde_tdemap.h b/contrib/pg_tde/src/include/access/pg_tde_tdemap.h index 471840554093e..dcf834c995fb3 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_tdemap.h +++ b/contrib/pg_tde/src/include/access/pg_tde_tdemap.h @@ -93,7 +93,7 @@ pg_tde_set_db_file_path(Oid dbOid, char *path) join_path_components(path, pg_tde_get_data_dir(), psprintf(PG_TDE_MAP_FILENAME, dbOid)); } -extern void pg_tde_save_smgr_key(RelFileLocator rel, const InternalKey *key, bool write_xlog); +extern void pg_tde_save_smgr_key(RelFileLocator rel, const InternalKey *key); extern bool pg_tde_has_smgr_key(RelFileLocator rel); extern InternalKey *pg_tde_get_smgr_key(RelFileLocator rel); extern void pg_tde_free_key_map_entry(RelFileLocator rel); diff --git a/contrib/pg_tde/src/smgr/pg_tde_smgr.c b/contrib/pg_tde/src/smgr/pg_tde_smgr.c index bed0d8e5fb61f..67e44e9d36786 100644 --- a/contrib/pg_tde/src/smgr/pg_tde_smgr.c +++ b/contrib/pg_tde/src/smgr/pg_tde_smgr.c @@ -3,7 +3,9 @@ #include "smgr/pg_tde_smgr.h" #include "storage/smgr.h" #include "storage/md.h" +#include "access/xloginsert.h" #include "catalog/catalog.h" +#include "access/pg_tde_xlog.h" #include "encryption/enc_aes.h" #include "encryption/enc_tde.h" #include "access/pg_tde_tdemap.h" @@ -77,11 +79,23 @@ tde_smgr_create_key(const RelFileLocatorBackend *smgr_rlocator) if (RelFileLocatorBackendIsTemp(*smgr_rlocator)) tde_smgr_save_temp_key(&smgr_rlocator->locator, key); else - pg_tde_save_smgr_key(smgr_rlocator->locator, key, true); + pg_tde_save_smgr_key(smgr_rlocator->locator, key); return key; } +static void +tde_smgr_log_create_key(const RelFileLocatorBackend *smgr_rlocator) +{ + XLogRelKey xlrec = { + .rlocator = smgr_rlocator->locator, + }; + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xlrec)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY); +} + void tde_smgr_create_key_redo(const RelFileLocator *rlocator) { @@ -92,7 +106,7 @@ tde_smgr_create_key_redo(const RelFileLocator *rlocator) pg_tde_generate_internal_key(&key, TDE_KEY_TYPE_SMGR); - pg_tde_save_smgr_key(*rlocator, &key, false); + pg_tde_save_smgr_key(*rlocator, &key); } static bool @@ -370,6 +384,7 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool else { key = tde_smgr_create_key(&reln->smgr_rlocator); + tde_smgr_log_create_key(&reln->smgr_rlocator); } tdereln->encryption_status = RELATION_KEY_AVAILABLE; From 45303ebba753482582333adb862f2595370502d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20=C3=85strand?= Date: Mon, 9 Jun 2025 12:20:02 +0200 Subject: [PATCH 3/3] Delete relation keys before create There are crash scenarios where keys are left behind in the key file even though the OID for the table goes unused. This meant that we could have keys laying around for newly created plaintext relations after OID wraparound. Simply removing any existing keys when relations are created seems appriopriate. Also move creation of pg_tde data dir to library init. This directory is used by the SMgr which is loaded regardless of whether any database are yet to create extension or not. --- contrib/pg_tde/src/access/pg_tde_xlog.c | 6 ++++ .../pg_tde/src/include/access/pg_tde_xlog.h | 1 + contrib/pg_tde/src/include/smgr/pg_tde_smgr.h | 1 + contrib/pg_tde/src/pg_tde.c | 3 +- contrib/pg_tde/src/smgr/pg_tde_smgr.c | 33 +++++++++++++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) diff --git a/contrib/pg_tde/src/access/pg_tde_xlog.c b/contrib/pg_tde/src/access/pg_tde_xlog.c index 0d0581d0604a3..43407ee3698f6 100644 --- a/contrib/pg_tde/src/access/pg_tde_xlog.c +++ b/contrib/pg_tde/src/access/pg_tde_xlog.c @@ -61,6 +61,12 @@ tdeheap_rmgr_redo(XLogReaderState *record) pg_tde_save_principal_key_redo(mkey); } + else if (info == XLOG_TDE_REMOVE_RELATION_KEY) + { + XLogRelKey *xlrec = (XLogRelKey *) XLogRecGetData(record); + + tde_smgr_delete_key_redo(&xlrec->rlocator); + } else if (info == XLOG_TDE_ROTATE_PRINCIPAL_KEY) { XLogPrincipalKeyRotate *xlrec = (XLogPrincipalKeyRotate *) XLogRecGetData(record); diff --git a/contrib/pg_tde/src/include/access/pg_tde_xlog.h b/contrib/pg_tde/src/include/access/pg_tde_xlog.h index 3938ae7763792..2f08fecb37162 100644 --- a/contrib/pg_tde/src/include/access/pg_tde_xlog.h +++ b/contrib/pg_tde/src/include/access/pg_tde_xlog.h @@ -17,6 +17,7 @@ #define XLOG_TDE_ROTATE_PRINCIPAL_KEY 0x20 #define XLOG_TDE_WRITE_KEY_PROVIDER 0x30 #define XLOG_TDE_INSTALL_EXTENSION 0x40 +#define XLOG_TDE_REMOVE_RELATION_KEY 0x50 /* ID 140 is registered for Percona TDE extension: https://wiki.postgresql.org/wiki/CustomWALResourceManagers */ #define RM_TDERMGR_ID 140 diff --git a/contrib/pg_tde/src/include/smgr/pg_tde_smgr.h b/contrib/pg_tde/src/include/smgr/pg_tde_smgr.h index e0f09efc4c39b..31a0d525f4fe6 100644 --- a/contrib/pg_tde/src/include/smgr/pg_tde_smgr.h +++ b/contrib/pg_tde/src/include/smgr/pg_tde_smgr.h @@ -14,6 +14,7 @@ extern void RegisterStorageMgr(void); extern void tde_smgr_create_key_redo(const RelFileLocator *rlocator); +extern void tde_smgr_delete_key_redo(const RelFileLocator *rlocator); extern bool tde_smgr_rel_is_encrypted(SMgrRelation reln); #endif /* PG_TDE_SMGR_H */ diff --git a/contrib/pg_tde/src/pg_tde.c b/contrib/pg_tde/src/pg_tde.c index 3127c2c4cff09..410c7ce7cb679 100644 --- a/contrib/pg_tde/src/pg_tde.c +++ b/contrib/pg_tde/src/pg_tde.c @@ -93,6 +93,7 @@ _PG_init(void) check_percona_api_version(); + pg_tde_init_data_dir(); AesInit(); TdeGucInit(); TdeEventCaptureInit(); @@ -113,8 +114,6 @@ _PG_init(void) static void extension_install(Oid databaseId) { - /* Initialize the TDE dir */ - pg_tde_init_data_dir(); key_provider_startup_cleanup(databaseId); principal_key_startup_cleanup(databaseId); } diff --git a/contrib/pg_tde/src/smgr/pg_tde_smgr.c b/contrib/pg_tde/src/smgr/pg_tde_smgr.c index 67e44e9d36786..476a30369e16d 100644 --- a/contrib/pg_tde/src/smgr/pg_tde_smgr.c +++ b/contrib/pg_tde/src/smgr/pg_tde_smgr.c @@ -109,6 +109,26 @@ tde_smgr_create_key_redo(const RelFileLocator *rlocator) pg_tde_save_smgr_key(*rlocator, &key); } +static void +tde_smgr_delete_key(const RelFileLocatorBackend *smgr_rlocator) +{ + XLogRelKey xlrec = { + .rlocator = smgr_rlocator->locator, + }; + + pg_tde_free_key_map_entry(smgr_rlocator->locator); + + XLogBeginInsert(); + XLogRegisterData((char *) &xlrec, sizeof(xlrec)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_REMOVE_RELATION_KEY); +} + +void +tde_smgr_delete_key_redo(const RelFileLocator *rlocator) +{ + pg_tde_free_key_map_entry(*rlocator); +} + static bool tde_smgr_is_encrypted(const RelFileLocatorBackend *smgr_rlocator) { @@ -363,6 +383,19 @@ tde_mdcreate(RelFileLocator relold, SMgrRelation reln, ForkNumber forknum, bool return; } + if (!isRedo) + { + /* + * If we have a key for this relation already, we need to remove it. + * This can happen if OID is re-used after a crash left a key for a + * non-existing relation in the key file. + * + * If we're in redo, a separate WAL record will make sure the key is + * removed. + */ + tde_smgr_delete_key(&reln->smgr_rlocator); + } + if (!tde_smgr_should_encrypt(&reln->smgr_rlocator, &relold)) { tdereln->encryption_status = RELATION_NOT_ENCRYPTED;