From b406bf53a2b303d83d500db37d6b8b9f7f37754e Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Wed, 4 Jun 2025 17:43:48 +0200 Subject: [PATCH 1/4] PG-1607 Add tests for that Vault CA path is optional for pg_tde_change_key_provider Also do not rely on the Vault server setup since any file on disk will do. --- .../pg_tde/t/pg_tde_change_key_provider.pl | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/contrib/pg_tde/t/pg_tde_change_key_provider.pl b/contrib/pg_tde/t/pg_tde_change_key_provider.pl index 284611d178498..f00e394840b74 100644 --- a/contrib/pg_tde/t/pg_tde_change_key_provider.pl +++ b/contrib/pg_tde/t/pg_tde_change_key_provider.pl @@ -22,7 +22,8 @@ q{SELECT oid FROM pg_catalog.pg_database WHERE datname = 'postgres'}); my $options; -my $token_file = $ENV{ROOT_TOKEN_FILE}; +my $token_file = "${PostgreSQL::Test::Utils::tmp_check}/vault_token"; +append_to_file($token_file, 'DUMMY'); $node->stop; @@ -65,13 +66,13 @@ $db_oid, 'database-provider', 'vault-v2', - 'http://vault-server.example:8200/', + 'https://vault-server.example:8200/', $token_file, 'mount-path', '/tmp/ca_path', ], qr/Key provider updated successfully!/, - 'updates key provider to vault-v2 type'); + 'updates key provider to vault-v2 type with https'); $node->start; @@ -90,7 +91,7 @@ is($options->{tokenPath}, $token_file, 'tokenPath is set correctly for vault-v2 provider'); is( $options->{url}, - 'http://vault-server.example:8200/', + 'https://vault-server.example:8200/', 'url is set correctly for vault-v2 provider'); is($options->{mountPath}, 'mount-path', 'mount path is set correctly for vault-v2 provider'); @@ -99,6 +100,45 @@ $node->stop; +command_like( + [ + 'pg_tde_change_key_provider', + '-D' => $node->data_dir, + $db_oid, + 'database-provider', + 'vault-v2', + 'http://vault-server.example:8200/', + $token_file, + 'mount-path-2', + ], + qr/Key provider updated successfully!/, + 'updates key provider to vault-v2 type with http'); + +$node->start; + +is( $node->safe_psql( + 'postgres', + q{SELECT provider_type FROM pg_tde_list_all_database_key_providers() WHERE provider_name = 'database-provider'} + ), + 'vault-v2', + 'provider type is set to vault-v2'); + +$options = decode_json( + $node->safe_psql( + 'postgres', + q{SELECT options FROM pg_tde_list_all_database_key_providers() WHERE provider_name = 'database-provider'} + )); +is($options->{tokenPath}, $token_file, + 'tokenPath is set correctly for vault-v2 provider'); +is( $options->{url}, + 'http://vault-server.example:8200/', + 'url is set correctly for vault-v2 provider'); +is($options->{mountPath}, 'mount-path-2', + 'mount path is set correctly for vault-v2 provider'); +is($options->{caPath}, '', 'CA path is set correctly for vault-v2 provider'); + +$node->stop; + command_like( [ 'pg_tde_change_key_provider', From b58a4f2eb516c20b6d1ad68b5504d7a3a90c6175 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Fri, 30 May 2025 17:37:36 +0200 Subject: [PATCH 2/4] PG-1607 Add a very basic test for change Vault key provider This is a very basic test but before this there were no tests at all. --- contrib/pg_tde/expected/vault_v2_test.out | 3 +++ contrib/pg_tde/sql/vault_v2_test.sql | 3 +++ 2 files changed, 6 insertions(+) diff --git a/contrib/pg_tde/expected/vault_v2_test.out b/contrib/pg_tde/expected/vault_v2_test.out index 7042a6fa43642..0dc4a637b53ad 100644 --- a/contrib/pg_tde/expected/vault_v2_test.out +++ b/contrib/pg_tde/expected/vault_v2_test.out @@ -54,4 +54,7 @@ DROP TABLE test_enc; -- Creating provider fails if we can't connect to vault SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); ERROR: HTTP(S) request to keyring provider "will-not-work" failed +-- Changing provider fails if we can't connect to vault +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); +ERROR: HTTP(S) request to keyring provider "vault-v2" failed DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/vault_v2_test.sql b/contrib/pg_tde/sql/vault_v2_test.sql index d0cce4719fccd..78c8c6e434cf8 100644 --- a/contrib/pg_tde/sql/vault_v2_test.sql +++ b/contrib/pg_tde/sql/vault_v2_test.sql @@ -34,4 +34,7 @@ DROP TABLE test_enc; -- Creating provider fails if we can't connect to vault SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); +-- Changing provider fails if we can't connect to vault +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); + DROP EXTENSION pg_tde; From a9f2608bd726819f91549cfcb893ddac36bc80f5 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Fri, 30 May 2025 18:38:54 +0200 Subject: [PATCH 3/4] PG-1607 Run our test suite for Vault with HTTPS Since the dev mode of Vault supports generating HTTPs ceritficates we should just use that since in production everyone will use HTTPS we should run our tests with HTTPS too. --- ci_scripts/setup-keyring-servers.sh | 14 ++++++++------ contrib/pg_tde/expected/vault_v2_test.out | 17 ++++++++++++----- contrib/pg_tde/sql/vault_v2_test.sql | 17 ++++++++++++----- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/ci_scripts/setup-keyring-servers.sh b/ci_scripts/setup-keyring-servers.sh index 356d98b586ce9..962f859244339 100755 --- a/ci_scripts/setup-keyring-servers.sh +++ b/ci_scripts/setup-keyring-servers.sh @@ -17,12 +17,14 @@ cd .. echo $SCRIPT_DIR pykmip-server -f "$SCRIPT_DIR/../contrib/pg_tde/pykmip-server.conf" -l /tmp/kmip-server.log & -TV=$(mktemp) -{ exec >$TV; vault server -dev; } & +CLUSTER_INFO=$(mktemp) +vault server -dev -dev-tls -dev-cluster-json="$CLUSTER_INFO" > /dev/null & sleep 10 -export ROOT_TOKEN_FILE=$(mktemp) -cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n > $ROOT_TOKEN_FILE -echo "export ROOT_TOKEN_FILE=$ROOT_TOKEN_FILE" +export VAULT_ROOT_TOKEN_FILE=$(mktemp) +jq -r .root_token "$CLUSTER_INFO" > "$VAULT_ROOT_TOKEN_FILE" +export VAULT_CACERT_FILE=$(jq -r .ca_cert_path "$CLUSTER_INFO") +rm "$CLUSTER_INFO" if [ -v GITHUB_ACTIONS ]; then - echo "ROOT_TOKEN_FILE=$ROOT_TOKEN_FILE" >> $GITHUB_ENV + echo "VAULT_ROOT_TOKEN_FILE=$VAULT_ROOT_TOKEN_FILE" >> $GITHUB_ENV + echo "VAULT_CACERT_FILE=$VAULT_CACERT_FILE" >> $GITHUB_ENV fi diff --git a/contrib/pg_tde/expected/vault_v2_test.out b/contrib/pg_tde/expected/vault_v2_test.out index 0dc4a637b53ad..291d230dd1df9 100644 --- a/contrib/pg_tde/expected/vault_v2_test.out +++ b/contrib/pg_tde/expected/vault_v2_test.out @@ -1,6 +1,7 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -\getenv root_token_file ROOT_TOKEN_FILE -SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','http://127.0.0.1:8200','DUMMY-TOKEN',NULL); +\getenv root_token_file VAULT_ROOT_TOKEN_FILE +\getenv cacert_file VAULT_CACERT_FILE +SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','https://127.0.0.1:8200','DUMMY-TOKEN',:'cacert_file'); pg_tde_add_database_key_provider_vault_v2 ------------------------------------------- @@ -16,7 +17,7 @@ CREATE TABLE test_enc( ) USING tde_heap; ERROR: principal key not configured HINT: create one using pg_tde_set_key before using encrypted tables -SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token_file','http://127.0.0.1:8200','secret',NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token_file','https://127.0.0.1:8200','secret',:'cacert_file'); pg_tde_add_database_key_provider_vault_v2 ------------------------------------------- @@ -52,9 +53,15 @@ SELECT pg_tde_verify_key(); DROP TABLE test_enc; -- Creating provider fails if we can't connect to vault -SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'https://127.0.0.1:61', 'secret', :'cacert_file'); ERROR: HTTP(S) request to keyring provider "will-not-work" failed -- Changing provider fails if we can't connect to vault -SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'https://127.0.0.1:61', 'secret', :'cacert_file'); ERROR: HTTP(S) request to keyring provider "vault-v2" failed +-- HTTPS without cert fails +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'https://127.0.0.1:8200', 'secret', NULL); +ERROR: HTTP(S) request to keyring provider "vault-v2" failed +-- HTTP against HTTPS server fails +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'http://127.0.0.1:8200', 'secret', NULL); +ERROR: Listing secrets of "http://127.0.0.1:8200" at mountpoint "secret" failed DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/sql/vault_v2_test.sql b/contrib/pg_tde/sql/vault_v2_test.sql index 78c8c6e434cf8..a1f5a92233db2 100644 --- a/contrib/pg_tde/sql/vault_v2_test.sql +++ b/contrib/pg_tde/sql/vault_v2_test.sql @@ -1,8 +1,9 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -\getenv root_token_file ROOT_TOKEN_FILE +\getenv root_token_file VAULT_ROOT_TOKEN_FILE +\getenv cacert_file VAULT_CACERT_FILE -SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','http://127.0.0.1:8200','DUMMY-TOKEN',NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','https://127.0.0.1:8200','DUMMY-TOKEN',:'cacert_file'); -- FAILS SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-incorrect'); @@ -12,7 +13,7 @@ CREATE TABLE test_enc( PRIMARY KEY (id) ) USING tde_heap; -SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token_file','http://127.0.0.1:8200','secret',NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token_file','https://127.0.0.1:8200','secret',:'cacert_file'); SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-v2'); CREATE TABLE test_enc( @@ -32,9 +33,15 @@ SELECT pg_tde_verify_key(); DROP TABLE test_enc; -- Creating provider fails if we can't connect to vault -SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'https://127.0.0.1:61', 'secret', :'cacert_file'); -- Changing provider fails if we can't connect to vault -SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'https://127.0.0.1:61', 'secret', :'cacert_file'); + +-- HTTPS without cert fails +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'https://127.0.0.1:8200', 'secret', NULL); + +-- HTTP against HTTPS server fails +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'http://127.0.0.1:8200', 'secret', NULL); DROP EXTENSION pg_tde; From 94677a81e13c8846136ee8636b13aea8c93a28d8 Mon Sep 17 00:00:00 2001 From: Andreas Karlsson Date: Wed, 4 Jun 2025 16:39:05 +0200 Subject: [PATCH 4/4] PG-1607 Encourage using HTTPS for Vault in our documentation We should encourage HTTPS over HTTP in out examples. --- .../docs/global-key-provider-configuration/vault.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md index 71aaaf075a56d..9db21f325ec0f 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md @@ -31,9 +31,9 @@ The following example is for testing purposes only. Use secure tokens and proper SELECT pg_tde_add_global_key_provider_vault_v2( 'my-vault', '/path/to/token_file', - 'http://vault.vault.svc.cluster.local:8200', + 'https://vault.vault.svc.cluster.local:8200', 'secret/data', - NULL + '/path/to/ca_cert.pem' ); ```