diff --git a/ci_scripts/setup-keyring-servers.sh b/ci_scripts/setup-keyring-servers.sh index 356d98b586ce9..e60804f922311 100755 --- a/ci_scripts/setup-keyring-servers.sh +++ b/ci_scripts/setup-keyring-servers.sh @@ -17,12 +17,14 @@ cd .. echo $SCRIPT_DIR pykmip-server -f "$SCRIPT_DIR/../contrib/pg_tde/pykmip-server.conf" -l /tmp/kmip-server.log & -TV=$(mktemp) -{ exec >$TV; vault server -dev; } & +CLUSTER_INFO=$(mktemp) +vault server -dev -dev-tls -dev-cluster-json="$CLUSTER_INFO" > /dev/null & sleep 10 -export ROOT_TOKEN_FILE=$(mktemp) -cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n > $ROOT_TOKEN_FILE -echo "export ROOT_TOKEN_FILE=$ROOT_TOKEN_FILE" +export VAULT_ROOT_TOKEN_FILE=$(mktemp) +<"$CLUSTER_INFO" python3 -c 'import json, sys; print(json.load(sys.stdin)["root_token"])' > "$VAULT_ROOT_TOKEN_FILE" +export VAULT_CACERT_FILE=$(<"$CLUSTER_INFO" python3 -c 'import json, sys; print(json.load(sys.stdin)["ca_cert_path"])') +rm "$CLUSTER_INFO" if [ -v GITHUB_ACTIONS ]; then - echo "ROOT_TOKEN_FILE=$ROOT_TOKEN_FILE" >> $GITHUB_ENV + echo "VAULT_ROOT_TOKEN_FILE=$VAULT_ROOT_TOKEN_FILE" >> $GITHUB_ENV + echo "VAULT_CACERT_FILE=$VAULT_CACERT_FILE" >> $GITHUB_ENV fi diff --git a/contrib/pg_tde/README.md b/contrib/pg_tde/README.md index cdcb0bbe47fbc..93d65f68285e8 100644 --- a/contrib/pg_tde/README.md +++ b/contrib/pg_tde/README.md @@ -118,7 +118,8 @@ _See [Make Builds for Developers](https://github.com/percona/pg_tde/wiki/Make-bu 'vault-provider', '/path/to/token_file', 'https://your.vault.server', - 'secret', NULL); + 'secret', + '/path/to/ca_cert.pem'); -- For File key provider -- pg_tde_add_database_key_provider_file(provider_name, file_path); diff --git a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md index 71aaaf075a56d..9db21f325ec0f 100644 --- a/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md +++ b/contrib/pg_tde/documentation/docs/global-key-provider-configuration/vault.md @@ -31,9 +31,9 @@ The following example is for testing purposes only. Use secure tokens and proper SELECT pg_tde_add_global_key_provider_vault_v2( 'my-vault', '/path/to/token_file', - 'http://vault.vault.svc.cluster.local:8200', + 'https://vault.vault.svc.cluster.local:8200', 'secret/data', - NULL + '/path/to/ca_cert.pem' ); ``` diff --git a/contrib/pg_tde/expected/vault_v2_test.out b/contrib/pg_tde/expected/vault_v2_test.out index 7042a6fa43642..32e17a72f5aa0 100644 --- a/contrib/pg_tde/expected/vault_v2_test.out +++ b/contrib/pg_tde/expected/vault_v2_test.out @@ -1,6 +1,7 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -\getenv root_token_file ROOT_TOKEN_FILE -SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','http://127.0.0.1:8200','DUMMY-TOKEN',NULL); +\getenv root_token_file VAULT_ROOT_TOKEN_FILE +\getenv cacert_file VAULT_CACERT_FILE +SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','https://127.0.0.1:8200','DUMMY-TOKEN',:'cacert_file'); pg_tde_add_database_key_provider_vault_v2 ------------------------------------------- @@ -16,7 +17,7 @@ CREATE TABLE test_enc( ) USING tde_heap; ERROR: principal key not configured HINT: create one using pg_tde_set_key before using encrypted tables -SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token_file','http://127.0.0.1:8200','secret',NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token_file','https://127.0.0.1:8200','secret',:'cacert_file'); pg_tde_add_database_key_provider_vault_v2 ------------------------------------------- @@ -52,6 +53,15 @@ SELECT pg_tde_verify_key(); DROP TABLE test_enc; -- Creating provider fails if we can't connect to vault -SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'https://127.0.0.1:61', 'secret', :'cacert_file'); ERROR: HTTP(S) request to keyring provider "will-not-work" failed +-- Changing provider fails if we can't connect to vault +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'https://127.0.0.1:61', 'secret', :'cacert_file'); +ERROR: HTTP(S) request to keyring provider "vault-v2" failed +-- HTTPS without cert fails +SELECT pg_tde_change_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','https://127.0.0.1:8200','DUMMY-TOKEN'); +ERROR: HTTP(S) request to keyring provider "vault-incorrect" failed +-- HTTP against HTTPS server fails +SELECT pg_tde_change_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','http://127.0.0.1:8200','DUMMY-TOKEN'); +ERROR: Listing secrets of "http://127.0.0.1:8200" at mountpoint "DUMMY-TOKEN" failed DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/pg_tde--1.0-rc.sql b/contrib/pg_tde/pg_tde--1.0-rc.sql index 909543f5fe2eb..201dec62e90a9 100644 --- a/contrib/pg_tde/pg_tde--1.0-rc.sql +++ b/contrib/pg_tde/pg_tde--1.0-rc.sql @@ -34,7 +34,7 @@ CREATE FUNCTION pg_tde_add_database_key_provider_vault_v2(provider_name TEXT, vault_token_path TEXT, vault_url TEXT, vault_mount_path TEXT, - vault_ca_path TEXT) + vault_ca_path TEXT DEFAULT NULL) RETURNS VOID LANGUAGE SQL BEGIN ATOMIC @@ -117,7 +117,7 @@ CREATE FUNCTION pg_tde_add_global_key_provider_vault_v2(provider_name TEXT, vault_token_path TEXT, vault_url TEXT, vault_mount_path TEXT, - vault_ca_path TEXT) + vault_ca_path TEXT DEFAULT NULL) RETURNS VOID LANGUAGE SQL BEGIN ATOMIC @@ -180,7 +180,7 @@ CREATE FUNCTION pg_tde_change_database_key_provider_vault_v2(provider_name TEXT, vault_token_path TEXT, vault_url TEXT, vault_mount_path TEXT, - vault_ca_path TEXT) + vault_ca_path TEXT DEFAULT NULL) RETURNS VOID LANGUAGE SQL BEGIN ATOMIC @@ -243,7 +243,7 @@ CREATE FUNCTION pg_tde_change_global_key_provider_vault_v2(provider_name TEXT, vault_token_path TEXT, vault_url TEXT, vault_mount_path TEXT, - vault_ca_path TEXT) + vault_ca_path TEXT DEFAULT NULL) RETURNS VOID LANGUAGE SQL BEGIN ATOMIC diff --git a/contrib/pg_tde/sql/vault_v2_test.sql b/contrib/pg_tde/sql/vault_v2_test.sql index d0cce4719fccd..e12ec655c0f63 100644 --- a/contrib/pg_tde/sql/vault_v2_test.sql +++ b/contrib/pg_tde/sql/vault_v2_test.sql @@ -1,8 +1,9 @@ CREATE EXTENSION IF NOT EXISTS pg_tde; -\getenv root_token_file ROOT_TOKEN_FILE +\getenv root_token_file VAULT_ROOT_TOKEN_FILE +\getenv cacert_file VAULT_CACERT_FILE -SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','http://127.0.0.1:8200','DUMMY-TOKEN',NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','https://127.0.0.1:8200','DUMMY-TOKEN',:'cacert_file'); -- FAILS SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-incorrect'); @@ -12,7 +13,7 @@ CREATE TABLE test_enc( PRIMARY KEY (id) ) USING tde_heap; -SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token_file','http://127.0.0.1:8200','secret',NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2',:'root_token_file','https://127.0.0.1:8200','secret',:'cacert_file'); SELECT pg_tde_set_key_using_database_key_provider('vault-v2-key','vault-v2'); CREATE TABLE test_enc( @@ -32,6 +33,15 @@ SELECT pg_tde_verify_key(); DROP TABLE test_enc; -- Creating provider fails if we can't connect to vault -SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'http://127.0.0.1:61', 'secret', NULL); +SELECT pg_tde_add_database_key_provider_vault_v2('will-not-work', :'root_token_file', 'https://127.0.0.1:61', 'secret', :'cacert_file'); + +-- Changing provider fails if we can't connect to vault +SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', :'root_token_file', 'https://127.0.0.1:61', 'secret', :'cacert_file'); + +-- HTTPS without cert fails +SELECT pg_tde_change_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','https://127.0.0.1:8200','DUMMY-TOKEN'); + +-- HTTP against HTTPS server fails +SELECT pg_tde_change_database_key_provider_vault_v2('vault-incorrect',:'root_token_file','http://127.0.0.1:8200','DUMMY-TOKEN'); DROP EXTENSION pg_tde; diff --git a/contrib/pg_tde/t/pg_tde_change_key_provider.pl b/contrib/pg_tde/t/pg_tde_change_key_provider.pl index 284611d178498..f00e394840b74 100644 --- a/contrib/pg_tde/t/pg_tde_change_key_provider.pl +++ b/contrib/pg_tde/t/pg_tde_change_key_provider.pl @@ -22,7 +22,8 @@ q{SELECT oid FROM pg_catalog.pg_database WHERE datname = 'postgres'}); my $options; -my $token_file = $ENV{ROOT_TOKEN_FILE}; +my $token_file = "${PostgreSQL::Test::Utils::tmp_check}/vault_token"; +append_to_file($token_file, 'DUMMY'); $node->stop; @@ -65,13 +66,13 @@ $db_oid, 'database-provider', 'vault-v2', - 'http://vault-server.example:8200/', + 'https://vault-server.example:8200/', $token_file, 'mount-path', '/tmp/ca_path', ], qr/Key provider updated successfully!/, - 'updates key provider to vault-v2 type'); + 'updates key provider to vault-v2 type with https'); $node->start; @@ -90,7 +91,7 @@ is($options->{tokenPath}, $token_file, 'tokenPath is set correctly for vault-v2 provider'); is( $options->{url}, - 'http://vault-server.example:8200/', + 'https://vault-server.example:8200/', 'url is set correctly for vault-v2 provider'); is($options->{mountPath}, 'mount-path', 'mount path is set correctly for vault-v2 provider'); @@ -99,6 +100,45 @@ $node->stop; +command_like( + [ + 'pg_tde_change_key_provider', + '-D' => $node->data_dir, + $db_oid, + 'database-provider', + 'vault-v2', + 'http://vault-server.example:8200/', + $token_file, + 'mount-path-2', + ], + qr/Key provider updated successfully!/, + 'updates key provider to vault-v2 type with http'); + +$node->start; + +is( $node->safe_psql( + 'postgres', + q{SELECT provider_type FROM pg_tde_list_all_database_key_providers() WHERE provider_name = 'database-provider'} + ), + 'vault-v2', + 'provider type is set to vault-v2'); + +$options = decode_json( + $node->safe_psql( + 'postgres', + q{SELECT options FROM pg_tde_list_all_database_key_providers() WHERE provider_name = 'database-provider'} + )); +is($options->{tokenPath}, $token_file, + 'tokenPath is set correctly for vault-v2 provider'); +is( $options->{url}, + 'http://vault-server.example:8200/', + 'url is set correctly for vault-v2 provider'); +is($options->{mountPath}, 'mount-path-2', + 'mount path is set correctly for vault-v2 provider'); +is($options->{caPath}, '', 'CA path is set correctly for vault-v2 provider'); + +$node->stop; + command_like( [ 'pg_tde_change_key_provider',