From 9a09a90bf7ddc4a84c61bc0f8d4c8380b8fd0dfc Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Mon, 5 Aug 2019 12:01:09 -0500 Subject: [PATCH 1/8] Revert "Revert "Merge pull request #382 from magento/MQE-1600"" This reverts commit 1359e26 --- composer.json | 4 +- .../FileStorageTest.php} | 17 +- etc/config/.credentials.example | 150 ++++++++--------- etc/config/.env.example | 4 + .../Handlers/CredentialStore.php | 154 ++++++------------ .../Handlers/SecretStorage/BaseStorage.php | 87 ++++++++++ .../Handlers/SecretStorage/FileStorage.php | 113 +++++++++++++ .../Handlers/SecretStorage/VaultStorage.php | 123 ++++++++++++++ .../SecretStorage/VaultTokenAuthStrategy.php | 47 ++++++ .../Test/Util/ActionMergeUtil.php | 3 +- .../Util/TestGenerator.php | 3 +- 11 files changed, 519 insertions(+), 186 deletions(-) rename dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/{CredentialStoreTest.php => SecretStorage/FileStorageTest.php} (59%) create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php create mode 100644 src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultTokenAuthStrategy.php diff --git a/composer.json b/composer.json index 39d206ee8..a37dd06c7 100755 --- a/composer.json +++ b/composer.json @@ -10,10 +10,12 @@ }, "require": { "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0", - "allure-framework/allure-codeception": "~1.3.0", "ext-curl": "*", + "allure-framework/allure-codeception": "~1.3.0", "codeception/codeception": "~2.3.4 || ~2.4.0 ", "consolidation/robo": "^1.0.0", + "csharpru/vault-php": "~3.5.3", + "csharpru/vault-php-guzzle6-transport": "^2.0", "flow/jsonpath": ">0.2", "fzaninotto/faker": "^1.6", "monolog/monolog": "^1.0", diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/CredentialStoreTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/FileStorageTest.php similarity index 59% rename from dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/CredentialStoreTest.php rename to dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/FileStorageTest.php index a451f8dc9..7e5824c8e 100644 --- a/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/CredentialStoreTest.php +++ b/dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/FileStorageTest.php @@ -4,33 +4,34 @@ * See COPYING.txt for license details. */ -namespace tests\unit\Magento\FunctionalTestFramework\DataGenerator\Handlers; +namespace tests\unit\Magento\FunctionalTestFramework\DataGenerator\Handlers\SecretStorage; -use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; use Magento\FunctionalTestingFramework\Util\MagentoTestCase; use AspectMock\Test as AspectMock; -class CredentialStoreTest extends MagentoTestCase +class FileStorageTest extends MagentoTestCase { /** - * Test basic encryption/decryption functionality in CredentialStore class. + * Test basic encryption/decryption functionality in FileStorage class. */ public function testBasicEncryptDecrypt() { - $testKey = 'myKey'; + $testKey = 'magento/myKey'; $testValue = 'myValue'; - AspectMock::double(CredentialStore::class, [ + AspectMock::double(FileStorage::class, [ 'readInCredentialsFile' => ["$testKey=$testValue"] ]); - $encryptedCred = CredentialStore::getInstance()->getSecret($testKey); + $fileStorage = new FileStorage(); + $encryptedCred = $fileStorage->getEncryptedValue($testKey); // assert the value we've gotten is in fact not identical to our test value $this->assertNotEquals($testValue, $encryptedCred); - $actualValue = CredentialStore::getInstance()->decryptSecretValue($encryptedCred); + $actualValue = $fileStorage->getDecryptedValue($encryptedCred); // assert that we are able to successfully decrypt our secret value $this->assertEquals($testValue, $actualValue); diff --git a/etc/config/.credentials.example b/etc/config/.credentials.example index ea8b03480..d9c73ac66 100644 --- a/etc/config/.credentials.example +++ b/etc/config/.credentials.example @@ -1,75 +1,75 @@ -#carriers/fedex/account= -#carriers/fedex/meter_number= -#carriers/fedex/key= -#carriers/fedex/password= - -#carriers/ups/password= -#carriers/ups/username= -#carriers/ups/access_license_number= -#carriers/ups/shipper_number= - -#carriers/usps/userid= -#carriers/usps/password= - -#carriers_dhl_id_us= -#carriers_dhl_password_us= -#carriers_dhl_account_us= - -#carriers_dhl_id_eu= -#carriers_dhl_password_eu= -#carriers_dhl_account_eu= - - -#payment_authorizenet_login= -#payment_authorizenet_trans_key= -#payment_authorizenet_trans_md5= - -#authorizenet_fraud_review_login= -#authorizenet_fraud_review_trans_key= -#authorizenet_fraud_review_md5= - -#braintree_enabled_fraud_merchant_account_id= -#braintree_enabled_fraud_merchant_id= -#braintree_enabled_fraud_public_key= -#braintree_enabled_fraud_private_key= - -#braintree_disabled_fraud_merchant_account_id= -#braintree_disabled_fraud_merchant_id= -#braintree_disabled_fraud_public_key= -#braintree_disabled_fraud_private_key= - -#payment/paypal_group_all_in_one/wpp_usuk/wpp_required_settings/wpp_and_express_checkout/business_account= -#payment/paypal_group_all_in_one/wpp_usuk/wpp_required_settings/wpp_and_express_checkout/api_username= -#payment/paypal_group_all_in_one/wpp_usuk/wpp_required_settings/wpp_and_express_checkout/api_password= -#payment/paypal_group_all_in_one/wpp_usuk/wpp_required_settings/wpp_and_express_checkout/api_signature= -#payment/paypal_express/merchant_id= - -#payflow_pro_fraud_protection_enabled_business_account= -#payflow_pro_fraud_protection_enabled_partner= -#payflow_pro_fraud_protection_enabled_user= -#payflow_pro_fraud_protection_enabled_pwd= -#payflow_pro_fraud_protection_enabled_vendor= - -#payflow_pro_business_account= -#payflow_pro_partner= -#payflow_pro_user= -#payflow_pro_pwd= -#payflow_pro_vendor= - -#payflow_link_business_account_email= -#payflow_link_partner= -#payflow_link_user= -#payflow_link_password= -#payflow_link_vendor= - -#payment/paypal_group_all_in_one/payments_pro_hosted_solution_with_express_checkout/pphs_required_settings/pphs_required_settings_pphs/business_account= -#payment/paypal_group_all_in_one/payments_pro_hosted_solution_with_express_checkout/pphs_required_settings/pphs_required_settings_pphs/api_username= -#payment/paypal_group_all_in_one/payments_pro_hosted_solution_with_express_checkout/pphs_required_settings/pphs_required_settings_pphs/api_password= -#payment/paypal_group_all_in_one/payments_pro_hosted_solution_with_express_checkout/pphs_required_settings/pphs_required_settings_pphs/api_signature= - -#payment/paypal_alternative_payment_methods/express_checkout_us/express_checkout_required/express_checkout_required_express_checkout/business_account= -#payment/paypal_alternative_payment_methods/express_checkout_us/express_checkout_required/express_checkout_required_express_checkout/api_username= -#payment/paypal_alternative_payment_methods/express_checkout_us/express_checkout_required/express_checkout_required_express_checkout/api_password= -#payment/paypal_alternative_payment_methods/express_checkout_us/express_checkout_required/express_checkout_required_express_checkout/api_signature= - -#fraud_protection/signifyd/api_key= \ No newline at end of file +#magento/magento/carriers_fedex_account= +#magento/carriers_fedex_meter_number= +#magento/carriers_fedex_key= +#magento/carriers_fedex_password= + +#magento/carriers_ups_password= +#magento/carriers_ups_username= +#magento/carriers_ups_access_license_number= +#magento/carriers_ups_shipper_number= + +#magento/carriers_usps_userid= +#magento/carriers_usps_password= + +#magento/carriers_dhl_id_us= +#magento/carriers_dhl_password_us= +#magento/carriers_dhl_account_us= + +#magento/carriers_dhl_id_eu= +#magento/carriers_dhl_password_eu= +#magento/carriers_dhl_account_eu= + + +#magento/payment_authorizenet_login= +#magento/payment_authorizenet_trans_key= +#magento/payment_authorizenet_trans_md5= + +#magento/authorizenet_fraud_review_login= +#magento/authorizenet_fraud_review_trans_key= +#magento/authorizenet_fraud_review_md5= + +#magento/braintree_enabled_fraud_merchant_account_id= +#magento/braintree_enabled_fraud_merchant_id= +#magento/braintree_enabled_fraud_public_key= +#magento/braintree_enabled_fraud_private_key= + +#magento/braintree_disabled_fraud_merchant_account_id= +#magento/braintree_disabled_fraud_merchant_id= +#magento/braintree_disabled_fraud_public_key= +#magento/braintree_disabled_fraud_private_key= + +#magento/payment_paypal_group_all_in_one_wpp_usuk_wpp_required_settings_wpp_and_express_checkout_business_account= +#magento/payment_paypal_group_all_in_one_wpp_usuk_wpp_required_settings_wpp_and_express_checkout_api_username= +#magento/payment_paypal_group_all_in_one_wpp_usuk_wpp_required_settings_wpp_and_express_checkout_api_password= +#magento/payment_paypal_group_all_in_one_wpp_usuk_wpp_required_settings_wpp_and_express_checkout_api_signature= +#magento/payment_paypal_express_merchant_id= + +#magento/payflow_pro_fraud_protection_enabled_business_account= +#magento/payflow_pro_fraud_protection_enabled_partner= +#magento/payflow_pro_fraud_protection_enabled_user= +#magento/payflow_pro_fraud_protection_enabled_pwd= +#magento/payflow_pro_fraud_protection_enabled_vendor= + +#magento/payflow_pro_business_account= +#magento/payflow_pro_partner= +#magento/payflow_pro_user= +#magento/payflow_pro_pwd= +#magento/payflow_pro_vendor= + +#magento/payflow_link_business_account_email= +#magento/payflow_link_partner= +#magento/payflow_link_user= +#magento/payflow_link_password= +#magento/payflow_link_vendor= + +#magento/payment_paypal_group_all_in_one_payments_pro_hosted_solution_with_express_checkout_pphs_required_settings_pphs_required_settings_pphs_business_account= +#magento/payment_paypal_group_all_in_one_payments_pro_hosted_solution_with_express_checkout_pphs_required_settings_pphs_required_settings_pphs_api_username= +#magento/payment_paypal_group_all_in_one_payments_pro_hosted_solution_with_express_checkout_pphs_required_settings_pphs_required_settings_pphs_api_password= +#magento/payment_paypal_group_all_in_one_payments_pro_hosted_solution_with_express_checkout_pphs_required_settings_pphs_required_settings_pphs_api_signature= + +#magento/payment_paypal_alternative_payment_methods_express_checkout_us_express_checkout_required_express_checkout_required_express_checkout_business_account= +#magento/payment_paypal_alternative_payment_methods_express_checkout_us_express_checkout_required_express_checkout_required_express_checkout_api_username= +#magento/payment_paypal_alternative_payment_methods_express_checkout_us_express_checkout_required_express_checkout_required_express_checkout_api_password= +#magento/payment_paypal_alternative_payment_methods_express_checkout_us_express_checkout_required_express_checkout_required_express_checkout_api_signature= + +#magento/fraud_protection_signifyd_api_key= \ No newline at end of file diff --git a/etc/config/.env.example b/etc/config/.env.example index 5dc7168be..cc82ae447 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -30,6 +30,10 @@ BROWSER=chrome #MAGENTO_RESTAPI_SERVER_PORT=8080 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https +#*** Uncomment and set vault base url and access token if you want to use vault to manage _CREDS secrets ***# +#CREDENTIAL_VAULT_BASE_URL= +#CREDENTIAL_VAULT_TOKEN= + #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= #FW_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index a83540683..016360752 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -6,47 +6,34 @@ namespace Magento\FunctionalTestingFramework\DataGenerator\Handlers; -use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; -use Magento\FunctionalTestingFramework\Console\BuildProjectCommand; use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException; -use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\FileStorage; +use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\VaultStorage; class CredentialStore { - const ENCRYPTION_ALGO = "AES-256-CBC"; + const ARRAY_KEY_FOR_VAULT = 'vault'; + const ARRAY_KEY_FOR_FILE = 'file'; /** - * Singleton instance - * - * @var CredentialStore - */ - private static $INSTANCE = null; - - /** - * Initial vector for open_ssl encryption. + * Credential storage array * - * @var string - */ - private $iv = null; - - /** - * Key for open_ssl encryption/decryption - * - * @var string + * @var array */ - private $encodedKey = null; + private $credStorage = []; /** - * Key/Value paris of credential names and their corresponding values + * Singleton instance * - * @var array + * @var CredentialStore */ - private $credentials = []; + private static $INSTANCE = null; /** * Static singleton getter for CredentialStore Instance * * @return CredentialStore + * @throws TestFrameworkException */ public static function getInstance() { @@ -58,120 +45,87 @@ public static function getInstance() } /** - * CredentialStore constructor. - */ - private function __construct() - { - $this->encodedKey = base64_encode(openssl_random_pseudo_bytes(16)); - $this->iv = substr(hash('sha256', $this->encodedKey), 0, 16); - $creds = $this->readInCredentialsFile(); - $this->credentials = $this->encryptCredFileContents($creds); - } - - /** - * Returns the value of a secret based on corresponding key + * CredentialStore constructor * - * @param string $key - * @return string|null * @throws TestFrameworkException */ - public function getSecret($key) + private function __construct() { - if (!array_key_exists($key, $this->credentials)) { - throw new TestFrameworkException( - "{$key} not defined in .credentials, please provide a value in order to use this secret in a test." - ); + // Initialize file storage + try { + $this->credStorage[self::ARRAY_KEY_FOR_FILE] = new FileStorage(); + } catch (TestFrameworkException $e) { } - // log here for verbose config - if (MftfApplicationConfig::getConfig()->verboseEnabled()) { - LoggingUtil::getInstance()->getLogger(CredentialStore::class)->debug( - "retrieving secret for key name {$key}" - ); + // Initialize vault storage + $csBaseUrl = getenv('CREDENTIAL_VAULT_BASE_URL'); + $csToken = getenv('CREDENTIAL_VAULT_TOKEN'); + if ($csBaseUrl !== false && $csToken !== false) { + try { + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( + rtrim($csBaseUrl, '/'), + $csToken + ); + } catch (TestFrameworkException $e) { + } } - return $this->credentials[$key] ?? null; - } - - /** - * Private function which reads in secret key/values from .credentials file and stores in memory as key/value pair. - * - * @return array - * @throws TestFrameworkException - */ - private function readInCredentialsFile() - { - $credsFilePath = str_replace( - '.credentials.example', - '.credentials', - BuildProjectCommand::CREDENTIALS_FILE_PATH - ); - - if (!file_exists($credsFilePath)) { + if (empty($this->credStorage)) { throw new TestFrameworkException( - "Cannot find .credentials file, please create in " - . TESTS_BP . " in order to reference sensitive information" + "No credential storage is properly configured. Please configure vault or .credentials file." ); } - - return file($credsFilePath, FILE_IGNORE_NEW_LINES); } /** - * Function which takes the contents of the credentials file and encrypts the entries. + * Get encrypted value by key * - * @param array $credContents - * @return array + * @param string $key + * @return string|null + * @throws TestFrameworkException */ - private function encryptCredFileContents($credContents) + public function getSecret($key) { - $encryptedCreds = []; - foreach ($credContents as $credValue) { - if (substr($credValue, 0, 1) === '#' || empty($credValue)) { - continue; - } - - list($key, $value) = explode("=", $credValue, 2); - if (!empty($value)) { - $encryptedCreds[$key] = openssl_encrypt( - $value, - self::ENCRYPTION_ALGO, - $this->encodedKey, - 0, - $this->iv - ); + // Get secret data from storage according to the order they are stored + // File storage is preferred over vault storage to allow local secret value overriding remote secret value + foreach ($this->credStorage as $storage) { + $value = $storage->getEncryptedValue($key); + if (null !== $value) { + return $value; } } - return $encryptedCreds; + throw new TestFrameworkException( + "\"{$key}\" not defined in vault or .credentials file, " + . "please provide a value in order to use this secret in a test." + ); } /** - * Takes a value encrypted at runtime and descrypts using the object's initial vector. + * Return decrypted input value * * @param string $value * @return string */ public function decryptSecretValue($value) { - return openssl_decrypt($value, self::ENCRYPTION_ALGO, $this->encodedKey, 0, $this->iv); + // Loop through storage to decrypt value + foreach ($this->credStorage as $storage) { + return $storage->getDecryptedValue($value); + } } /** - * Takes a string that contains encrypted data at runtime and decrypts each value. + * Return decrypted values for all occurrences from input string * * @param string $string * @return mixed */ public function decryptAllSecretsInString($string) { - $newString = $string; - foreach ($this->credentials as $name => $secretValue) { - if (strpos($newString, $secretValue) !== false) { - $decryptedValue = $this->decryptSecretValue($secretValue); - $newString = str_replace($secretValue, $decryptedValue, $newString); - } + // Loop through storage to decrypt all occurrences from input string + foreach ($this->credStorage as $storage) { + return $storage->getAllDecryptedValuesInString($string); } - return $newString; } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php new file mode 100644 index 000000000..cb892a545 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/BaseStorage.php @@ -0,0 +1,87 @@ + $secretValue) { + if (strpos($newString, $secretValue) !== false) { + $decryptedValue = self::getDecryptedValue($secretValue); + $newString = str_replace($secretValue, $decryptedValue, $newString); + } + } + return $newString; + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php new file mode 100644 index 000000000..064610c79 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/FileStorage.php @@ -0,0 +1,113 @@ +readInCredentialsFile(); + $this->secretData = $this->encryptCredFileContents($creds); + } + + /** + * Returns the value of a secret based on corresponding key + * + * @param string $key + * @return string|null + */ + public function getEncryptedValue($key) + { + $value = null; + // Check if secret is in cached array + if (null !== ($value = parent::getEncryptedValue($key))) { + return $value; + } + + // log here for verbose config + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(FileStorage::class)->debug( + "retrieving secret for key name {$key} from file" + ); + } + + // Retrieve from file storage + if (array_key_exists($key, $this->secretData) && (null !== ($value = $this->secretData[$key]))) { + parent::$cachedSecretData[$key] = $value; + } + + return $value; + } + + /** + * Private function which reads in secret key/values from .credentials file and stores in memory as key/value pair + * + * @return array + * @throws TestFrameworkException + */ + private function readInCredentialsFile() + { + $credsFilePath = str_replace( + '.credentials.example', + '.credentials', + BuildProjectCommand::CREDENTIALS_FILE_PATH + ); + + if (!file_exists($credsFilePath)) { + throw new TestFrameworkException( + "Credential file is not used: .credentials file not found in " . TESTS_BP + ); + } + + return file($credsFilePath, FILE_IGNORE_NEW_LINES); + } + + /** + * Function which takes the contents of the credentials file and encrypts the entries + * + * @param array $credContents + * @return array + */ + private function encryptCredFileContents($credContents) + { + $encryptedCreds = []; + foreach ($credContents as $credValue) { + if (substr($credValue, 0, 1) === '#' || empty($credValue)) { + continue; + } + + list($key, $value) = explode("=", $credValue, 2); + if (!empty($value)) { + $encryptedCreds[$key] = openssl_encrypt( + $value, + parent::ENCRYPTION_ALGO, + parent::$encodedKey, + 0, + parent::$iv + ); + } + } + return $encryptedCreds; + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php new file mode 100644 index 000000000..de80b83e3 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -0,0 +1,123 @@ +client) { + // Creating the client using Guzzle6 Transport and passing a custom url + $this->client = new Client(new Guzzle6Transport(['base_uri' => $baseUrl])); + } + $this->token = $token; + if (!$this->authenticated()) { + throw new TestFrameworkException("Credential vault is not used: cannot authenticate"); + } + } + + /** + * Returns the value of a secret based on corresponding key + * + * @param string $key + * @return string|null + */ + public function getEncryptedValue($key) + { + // Check if secret is in cached array + if (null !== ($value = parent::getEncryptedValue($key))) { + return $value; + } + + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "Retrieving secret for key name {$key} from vault" + ); + } + + $reValue = null; + try { + // Split vendor/key to construct secret path + list($vendor, $key) = explode('/', trim($key, '/'), 2); + $url = self::BASE_PATH + . (empty(self::KV_DATA) ? '' : '/' . self::KV_DATA) + . self::MFTF_PATH + . '/' + . $vendor + . '/' + . $key; + // Read value by key from vault + $value = $this->client->read($url)->getData()[self::KV_DATA][$key]; + // Encrypt value for return + $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); + parent::$cachedSecretData[$key] = $reValue; + } catch (\Exception $e) { + if (MftfApplicationConfig::getConfig()->verboseEnabled()) { + LoggingUtil::getInstance()->getLogger(VaultStorage::class)->debug( + "Unable to read secret for key name {$key} from vault" + ); + } + } + return $reValue; + } + + /** + * Check if vault token is valid + * + * @return boolean + */ + private function authenticated() + { + try { + // Authenticating using token auth backend + $authenticated = $this->client + ->setAuthenticationStrategy(new VaultTokenAuthStrategy($this->token)) + ->authenticate(); + + if ($authenticated) { + return true; + } + } catch (\Exception $e) { + } + return false; + } +} diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultTokenAuthStrategy.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultTokenAuthStrategy.php new file mode 100644 index 000000000..716344ca1 --- /dev/null +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultTokenAuthStrategy.php @@ -0,0 +1,47 @@ +token = $token; + } + + /** + * Returns auth for further interactions with Vault + * + * @return Auth + * @throws TestFrameworkException + */ + public function authenticate() + { + try { + return new Auth(['clientToken' => $this->token]); + } catch (\Exception $e) { + throw new TestFrameworkException("Cannot authenticate Vault token."); + } + } +} diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/ActionMergeUtil.php b/src/Magento/FunctionalTestingFramework/Test/Util/ActionMergeUtil.php index 2a939a8c7..cd81ade24 100644 --- a/src/Magento/FunctionalTestingFramework/Test/Util/ActionMergeUtil.php +++ b/src/Magento/FunctionalTestingFramework/Test/Util/ActionMergeUtil.php @@ -31,6 +31,7 @@ class ActionMergeUtil const DEFAULT_WAIT_ORDER = 'after'; const APPROVED_ACTIONS = ['fillField', 'magentoCLI', 'field']; const SECRET_MAPPING = ['fillField' => 'fillSecretField', 'magentoCLI' => 'magentoCLISecret']; + const CREDS_REGEX = "/{{_CREDS\.([\w|\/]+)}}/"; /** * Array holding final resulting steps @@ -149,7 +150,7 @@ private function actionAttributeContainsSecretRef($actionAttributes) return $this->actionAttributeContainsSecretRef($actionAttribute); } - preg_match_all("/{{_CREDS\.([\w]+)}}/", $actionAttribute, $matches); + preg_match_all(self::CREDS_REGEX, $actionAttribute, $matches); if (!empty($matches[0])) { return true; diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php index 415e22956..3f1417fd1 100644 --- a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php +++ b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php @@ -24,6 +24,7 @@ use Magento\FunctionalTestingFramework\Test\Util\ActionObjectExtractor; use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor; use Magento\FunctionalTestingFramework\Util\Filesystem\DirSetupUtil; +use Magento\FunctionalTestingFramework\Test\Util\ActionMergeUtil; /** * Class TestGenerator @@ -1904,7 +1905,7 @@ private function resolveAllRuntimeReferences($args) { $runtimeReferenceRegex = [ "/{{_ENV\.([\w]+)}}/" => 'getenv', - "/{{_CREDS\.([\w]+)}}/" => 'CredentialStore::getInstance()->getSecret' + ActionMergeUtil::CREDS_REGEX => 'CredentialStore::getInstance()->getSecret' ]; $argResult = $args; From 82371b1549756ae9450bcea29bb1d5b741b7e939 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Mon, 5 Aug 2019 13:06:49 -0500 Subject: [PATCH 2/8] MQE-1600: MFTF Vault integration - added composer.lock change --- composer.lock | 603 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 532 insertions(+), 71 deletions(-) diff --git a/composer.lock b/composer.lock index 7875ec7e1..5ce61bae1 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8c865770654a7053c62a505c2ca31942", + "content-hash": "39e380b2acfd8adcd2c3642655f5a2a1", "packages": [ { "name": "allure-framework/allure-codeception", @@ -168,6 +168,99 @@ ], "time": "2016-10-30T11:50:56+00:00" }, + { + "name": "cache/cache", + "version": "0.4.0", + "source": { + "type": "git", + "url": "https://github.com/php-cache/cache.git", + "reference": "902b2e5b54ea57e3a801437748652228c4c58604" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-cache/cache/zipball/902b2e5b54ea57e3a801437748652228c4c58604", + "reference": "902b2e5b54ea57e3a801437748652228c4c58604", + "shasum": "" + }, + "require": { + "doctrine/cache": "^1.3", + "league/flysystem": "^1.0", + "php": "^5.6 || ^7.0", + "psr/cache": "^1.0", + "psr/log": "^1.0", + "psr/simple-cache": "^1.0" + }, + "conflict": { + "cache/adapter-common": "*", + "cache/apc-adapter": "*", + "cache/apcu-adapter": "*", + "cache/array-adapter": "*", + "cache/chain-adapter": "*", + "cache/doctrine-adapter": "*", + "cache/filesystem-adapter": "*", + "cache/hierarchical-cache": "*", + "cache/illuminate-adapter": "*", + "cache/memcache-adapter": "*", + "cache/memcached-adapter": "*", + "cache/mongodb-adapter": "*", + "cache/predis-adapter": "*", + "cache/psr-6-doctrine-bridge": "*", + "cache/redis-adapter": "*", + "cache/session-handler": "*", + "cache/taggable-cache": "*", + "cache/void-adapter": "*" + }, + "require-dev": { + "cache/integration-tests": "^0.16", + "defuse/php-encryption": "^2.0", + "illuminate/cache": "^5.4", + "mockery/mockery": "^0.9", + "phpunit/phpunit": "^4.0 || ^5.1", + "predis/predis": "^1.0", + "symfony/cache": "dev-master" + }, + "suggest": { + "ext-apc": "APC extension is required to use the APC Adapter", + "ext-apcu": "APCu extension is required to use the APCu Adapter", + "ext-memcache": "Memcache extension is required to use the Memcache Adapter", + "ext-memcached": "Memcached extension is required to use the Memcached Adapter", + "ext-mongodb": "Mongodb extension required to use the Mongodb adapter", + "ext-redis": "Redis extension is required to use the Redis adapter", + "mongodb/mongodb": "Mongodb lib required to use the Mongodb adapter" + }, + "type": "library", + "autoload": { + "psr-4": { + "Cache\\": "src/" + }, + "exclude-from-classmap": [ + "**/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Aaron Scherer", + "email": "aequasi@gmail.com", + "homepage": "https://github.com/aequasi" + }, + { + "name": "Tobias Nyholm", + "email": "tobias.nyholm@gmail.com", + "homepage": "https://github.com/nyholm" + } + ], + "description": "Library of all the php-cache adapters", + "homepage": "http://www.php-cache.com/en/latest/", + "keywords": [ + "cache", + "psr6" + ], + "time": "2017-03-28T16:08:48+00:00" + }, { "name": "codeception/codeception", "version": "2.3.9", @@ -668,6 +761,92 @@ "homepage": "https://github.com/container-interop/container-interop", "time": "2017-02-14T19:40:03+00:00" }, + { + "name": "csharpru/vault-php", + "version": "3.5.3", + "source": { + "type": "git", + "url": "https://github.com/CSharpRU/vault-php.git", + "reference": "04be9776310fe7d1afb97795645f95c21e6b4fcf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/CSharpRU/vault-php/zipball/04be9776310fe7d1afb97795645f95c21e6b4fcf", + "reference": "04be9776310fe7d1afb97795645f95c21e6b4fcf", + "shasum": "" + }, + "require": { + "cache/cache": "^0.4.0", + "doctrine/inflector": "~1.1.0", + "guzzlehttp/promises": "^1.3", + "guzzlehttp/psr7": "^1.4", + "psr/cache": "^1.0", + "psr/log": "^1.0", + "weew/helpers-array": "^1.3" + }, + "require-dev": { + "codacy/coverage": "^1.1", + "codeception/codeception": "^2.2", + "csharpru/vault-php-guzzle6-transport": "~2.0", + "php-vcr/php-vcr": "^1.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "Vault\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Yaroslav Lukyanov", + "email": "c_sharp@mail.ru" + } + ], + "description": "Best Vault client for PHP that you can find", + "time": "2018-04-28T04:52:17+00:00" + }, + { + "name": "csharpru/vault-php-guzzle6-transport", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/CSharpRU/vault-php-guzzle6-transport.git", + "reference": "33c392120ac9f253b62b034e0e8ffbbdb3513bd8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/CSharpRU/vault-php-guzzle6-transport/zipball/33c392120ac9f253b62b034e0e8ffbbdb3513bd8", + "reference": "33c392120ac9f253b62b034e0e8ffbbdb3513bd8", + "shasum": "" + }, + "require": { + "guzzlehttp/guzzle": "~6.2", + "guzzlehttp/promises": "^1.3", + "guzzlehttp/psr7": "^1.4" + }, + "type": "library", + "autoload": { + "psr-4": { + "VaultTransports\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Yaroslav Lukyanov", + "email": "c_sharp@mail.ru" + } + ], + "description": "Guzzle6 transport for Vault PHP client", + "time": "2019-03-10T06:17:37+00:00" + }, { "name": "dflydev/dot-access-data", "version": "v1.1.0", @@ -795,6 +974,143 @@ ], "time": "2017-02-24T16:22:25+00:00" }, + { + "name": "doctrine/cache", + "version": "v1.6.2", + "source": { + "type": "git", + "url": "https://github.com/doctrine/cache.git", + "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/cache/zipball/eb152c5100571c7a45470ff2a35095ab3f3b900b", + "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b", + "shasum": "" + }, + "require": { + "php": "~5.5|~7.0" + }, + "conflict": { + "doctrine/common": ">2.2,<2.4" + }, + "require-dev": { + "phpunit/phpunit": "~4.8|~5.0", + "predis/predis": "~1.0", + "satooshi/php-coveralls": "~0.6" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.6.x-dev" + } + }, + "autoload": { + "psr-4": { + "Doctrine\\Common\\Cache\\": "lib/Doctrine/Common/Cache" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Roman Borschel", + "email": "roman@code-factory.org" + }, + { + "name": "Benjamin Eberlei", + "email": "kontakt@beberlei.de" + }, + { + "name": "Guilherme Blanco", + "email": "guilhermeblanco@gmail.com" + }, + { + "name": "Jonathan Wage", + "email": "jonwage@gmail.com" + }, + { + "name": "Johannes Schmitt", + "email": "schmittjoh@gmail.com" + } + ], + "description": "Caching library offering an object-oriented API for many cache backends", + "homepage": "http://www.doctrine-project.org", + "keywords": [ + "cache", + "caching" + ], + "time": "2017-07-22T12:49:21+00:00" + }, + { + "name": "doctrine/inflector", + "version": "v1.1.0", + "source": { + "type": "git", + "url": "https://github.com/doctrine/inflector.git", + "reference": "90b2128806bfde671b6952ab8bea493942c1fdae" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/doctrine/inflector/zipball/90b2128806bfde671b6952ab8bea493942c1fdae", + "reference": "90b2128806bfde671b6952ab8bea493942c1fdae", + "shasum": "" + }, + "require": { + "php": ">=5.3.2" + }, + "require-dev": { + "phpunit/phpunit": "4.*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1.x-dev" + } + }, + "autoload": { + "psr-0": { + "Doctrine\\Common\\Inflector\\": "lib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Roman Borschel", + "email": "roman@code-factory.org" + }, + { + "name": "Benjamin Eberlei", + "email": "kontakt@beberlei.de" + }, + { + "name": "Guilherme Blanco", + "email": "guilhermeblanco@gmail.com" + }, + { + "name": "Jonathan Wage", + "email": "jonwage@gmail.com" + }, + { + "name": "Johannes Schmitt", + "email": "schmittjoh@gmail.com" + } + ], + "description": "Common String Manipulations with regard to casing and singular/plural rules.", + "homepage": "http://www.doctrine-project.org", + "keywords": [ + "inflection", + "pluralize", + "singularize", + "string" + ], + "time": "2015-11-06T14:35:42+00:00" + }, { "name": "doctrine/instantiator", "version": "1.0.5", @@ -1608,6 +1924,90 @@ ], "time": "2017-05-10T09:20:27+00:00" }, + { + "name": "league/flysystem", + "version": "1.0.53", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/flysystem.git", + "reference": "08e12b7628f035600634a5e76d95b5eb66cea674" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/08e12b7628f035600634a5e76d95b5eb66cea674", + "reference": "08e12b7628f035600634a5e76d95b5eb66cea674", + "shasum": "" + }, + "require": { + "ext-fileinfo": "*", + "php": ">=5.5.9" + }, + "conflict": { + "league/flysystem-sftp": "<1.0.6" + }, + "require-dev": { + "phpspec/phpspec": "^3.4", + "phpunit/phpunit": "^5.7.10" + }, + "suggest": { + "ext-fileinfo": "Required for MimeType", + "ext-ftp": "Allows you to use FTP server storage", + "ext-openssl": "Allows you to use FTPS server storage", + "league/flysystem-aws-s3-v2": "Allows you to use S3 storage with AWS SDK v2", + "league/flysystem-aws-s3-v3": "Allows you to use S3 storage with AWS SDK v3", + "league/flysystem-azure": "Allows you to use Windows Azure Blob storage", + "league/flysystem-cached-adapter": "Flysystem adapter decorator for metadata caching", + "league/flysystem-eventable-filesystem": "Allows you to use EventableFilesystem", + "league/flysystem-rackspace": "Allows you to use Rackspace Cloud Files", + "league/flysystem-sftp": "Allows you to use SFTP server storage via phpseclib", + "league/flysystem-webdav": "Allows you to use WebDAV storage", + "league/flysystem-ziparchive": "Allows you to use ZipArchive adapter", + "spatie/flysystem-dropbox": "Allows you to use Dropbox storage", + "srmklive/flysystem-dropbox-v2": "Allows you to use Dropbox storage for PHP 5 applications" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-4": { + "League\\Flysystem\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frank de Jonge", + "email": "info@frenky.net" + } + ], + "description": "Filesystem abstraction: Many filesystems, one API.", + "keywords": [ + "Cloud Files", + "WebDAV", + "abstraction", + "aws", + "cloud", + "copy.com", + "dropbox", + "file systems", + "files", + "filesystem", + "filesystems", + "ftp", + "rackspace", + "remote", + "s3", + "sftp", + "storage" + ], + "time": "2019-06-18T20:09:29+00:00" + }, { "name": "monolog/monolog", "version": "1.24.0", @@ -2679,6 +3079,52 @@ "abandoned": true, "time": "2018-08-09T05:50:03+00:00" }, + { + "name": "psr/cache", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/cache.git", + "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/cache/zipball/d11b50ad223250cf17b86e38383413f5a6764bf8", + "reference": "d11b50ad223250cf17b86e38383413f5a6764bf8", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Cache\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interface for caching libraries", + "keywords": [ + "cache", + "psr", + "psr-6" + ], + "time": "2016-08-06T20:24:11+00:00" + }, { "name": "psr/container", "version": "1.0.0", @@ -2825,6 +3271,54 @@ ], "time": "2016-10-10T12:19:37+00:00" }, + { + "name": "psr/simple-cache", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/simple-cache.git", + "reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/simple-cache/zipball/408d5eafb83c57f6365a3ca330ff23aa4a5fa39b", + "reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\SimpleCache\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interfaces for simple caching", + "keywords": [ + "cache", + "caching", + "psr", + "psr-16", + "simple-cache" + ], + "time": "2017-10-23T01:57:42+00:00" + }, { "name": "ramsey/uuid", "version": "3.8.0", @@ -4397,6 +4891,43 @@ "validate" ], "time": "2018-01-29T19:49:41+00:00" + }, + { + "name": "weew/helpers-array", + "version": "v1.3.1", + "source": { + "type": "git", + "url": "https://github.com/weew/helpers-array.git", + "reference": "9bff63111f9765b4277750db8d276d92b3e16ed0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/weew/helpers-array/zipball/9bff63111f9765b4277750db8d276d92b3e16ed0", + "reference": "9bff63111f9765b4277750db8d276d92b3e16ed0", + "shasum": "" + }, + "require-dev": { + "phpunit/phpunit": "^4.7", + "satooshi/php-coveralls": "^0.6.1" + }, + "type": "library", + "autoload": { + "files": [ + "src/array.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Maxim Kott", + "email": "maximkott@gmail.com" + } + ], + "description": "Useful collection of php array helpers.", + "time": "2016-07-21T11:18:01+00:00" } ], "packages-dev": [ @@ -4556,76 +5087,6 @@ "description": "Experimental Mocking Framework powered by Aspects", "time": "2018-10-07T16:21:11+00:00" }, - { - "name": "doctrine/cache", - "version": "v1.6.2", - "source": { - "type": "git", - "url": "https://github.com/doctrine/cache.git", - "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/doctrine/cache/zipball/eb152c5100571c7a45470ff2a35095ab3f3b900b", - "reference": "eb152c5100571c7a45470ff2a35095ab3f3b900b", - "shasum": "" - }, - "require": { - "php": "~5.5|~7.0" - }, - "conflict": { - "doctrine/common": ">2.2,<2.4" - }, - "require-dev": { - "phpunit/phpunit": "~4.8|~5.0", - "predis/predis": "~1.0", - "satooshi/php-coveralls": "~0.6" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.6.x-dev" - } - }, - "autoload": { - "psr-4": { - "Doctrine\\Common\\Cache\\": "lib/Doctrine/Common/Cache" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Roman Borschel", - "email": "roman@code-factory.org" - }, - { - "name": "Benjamin Eberlei", - "email": "kontakt@beberlei.de" - }, - { - "name": "Guilherme Blanco", - "email": "guilhermeblanco@gmail.com" - }, - { - "name": "Jonathan Wage", - "email": "jonwage@gmail.com" - }, - { - "name": "Johannes Schmitt", - "email": "schmittjoh@gmail.com" - } - ], - "description": "Caching library offering an object-oriented API for many cache backends", - "homepage": "http://www.doctrine-project.org", - "keywords": [ - "cache", - "caching" - ], - "time": "2017-07-22T12:49:21+00:00" - }, { "name": "gitonomy/gitlib", "version": "v1.0.4", From 56f8ac0d278ce2851fb63eb1f351c6525a624098 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 24 Jul 2019 11:41:30 -0500 Subject: [PATCH 3/8] MQE-1647: read vault token from local file system (cherry picked from commit 9f63033) --- .../Handlers/CredentialStore.php | 8 +- .../Handlers/SecretStorage/VaultStorage.php | 98 ++++++++++++++++++- 2 files changed, 96 insertions(+), 10 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 016360752..75707b941 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -59,13 +59,9 @@ private function __construct() // Initialize vault storage $csBaseUrl = getenv('CREDENTIAL_VAULT_BASE_URL'); - $csToken = getenv('CREDENTIAL_VAULT_TOKEN'); - if ($csBaseUrl !== false && $csToken !== false) { + if ($csBaseUrl !== false) { try { - $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( - rtrim($csBaseUrl, '/'), - $csToken - ); + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage(rtrim($csBaseUrl, '/')); } catch (TestFrameworkException $e) { } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index de80b83e3..415627cda 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -21,6 +21,21 @@ class VaultStorage extends BaseStorage const BASE_PATH = '/dx_magento_qe'; const KV_DATA = 'data'; + /** + * Default vault token file + */ + const TOKEN_FILE = '.vault-token'; + /** + * Default vault config file + */ + const CONFIG_FILE = '.vault'; + /** + * Environment variable name for vault config path + */ + const CONFIG_PATH_ENV_VAR = 'VAULT_CONFIG_PATH'; + + const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(.+)$~"; + /** * Vault client * @@ -33,23 +48,22 @@ class VaultStorage extends BaseStorage * * @var string */ - private $token; + private $token = null; /** * CredentialVault constructor * * @param string $baseUrl - * @param string $token * @throws TestFrameworkException */ - public function __construct($baseUrl, $token) + public function __construct($baseUrl) { parent::__construct(); if (null === $this->client) { // Creating the client using Guzzle6 Transport and passing a custom url $this->client = new Client(new Guzzle6Transport(['base_uri' => $baseUrl])); } - $this->token = $token; + $this->readVaultTokenFromFileSystem(); if (!$this->authenticated()) { throw new TestFrameworkException("Credential vault is not used: cannot authenticate"); } @@ -120,4 +134,80 @@ private function authenticated() } return false; } + + /** + * Read vault token from file system + * + * @return void + * @throws TestFrameworkException + */ + private function readVaultTokenFromFileSystem() + { + // Find user home directory + $homeDir = getenv('HOME'); + if ($homeDir === false) { + // If HOME is not set, don't fail right away + $homeDir = '~/'; + } else { + $homeDir = rtrim($homeDir, '/') . '/'; + } + + $vaultTokenFile = $homeDir . self::TOKEN_FILE; + if (file_exists($vaultTokenFile)) { + // Found .vault-token file in default location, construct command + $cmd = 'cat ' . $vaultTokenFile; + } else { + // Otherwise search vault config file for custom token helper script + $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); + if ($vaultConfigPath === false) { + $vaultConfigFile = $homeDir . self::CONFIG_FILE; + } else { + $vaultConfigFile = rtrim($vaultConfigPath, '/') . '/' . self::CONFIG_FILE; + } + // Found .vault config file, read custom token helper script and construct command + if (file_exists($vaultConfigFile) + && !empty($cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)))) { + $cmd = $cmd . ' get'; + } else { + throw new TestFrameworkException( + 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' + ); + } + } + $this->token = $this->execVaultTokenHelper($cmd); + } + + /** + * Get vault token helper script by parsing lines in vault config file + * + * @param array $lines + * @return array + */ + private function getTokenHelperScript($lines) + { + $tokenHelper = ''; + foreach ($lines as $line) { + preg_match(self::TOKEN_HELPER_REGEX, $line, $matches); + if (isset($matches[1])) { + $tokenHelper = trim(trim(trim($matches[1]), '"')); + } + } + return $tokenHelper; + } + + /** + * Execute vault token helper script and return the token it contains + * + * @param string $cmd + * @return string + */ + private function execVaultTokenHelper($cmd) + { + $output = ''; + exec($cmd, $out, $status); + if ($status === 0 && isset($out[0])) { + $output = $out[0]; + } + return $output; + } } From 242045c8a02cddc533ba0f262448d2c531dc4a5d Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 24 Jul 2019 12:01:00 -0500 Subject: [PATCH 4/8] MQE-1647: read vault token from local file system (cherry picked from commit 2ddaa61) --- .../Handlers/SecretStorage/VaultStorage.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 415627cda..bb88b18c5 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -181,7 +181,7 @@ private function readVaultTokenFromFileSystem() * Get vault token helper script by parsing lines in vault config file * * @param array $lines - * @return array + * @return string */ private function getTokenHelperScript($lines) { @@ -200,14 +200,16 @@ private function getTokenHelperScript($lines) * * @param string $cmd * @return string + * @throws TestFrameworkException */ private function execVaultTokenHelper($cmd) { - $output = ''; exec($cmd, $out, $status); - if ($status === 0 && isset($out[0])) { - $output = $out[0]; + if ($status === 0 && isset($out[0]) && !empty($out[0])) { + return $out[0]; } - return $output; + throw new TestFrameworkException( + 'Error running custom vault token helper script. Please make sure vault CLI works in your environment.' + ); } } From 7d1104202bd3e709f51ac6d598d74f0076dfbf2c Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Thu, 25 Jul 2019 09:04:51 -0500 Subject: [PATCH 5/8] MQE-1647: read vault token from local file system (cherry picked from commit 599d8fd) --- etc/config/.env.example | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/config/.env.example b/etc/config/.env.example index cc82ae447..f091688f5 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -32,7 +32,6 @@ BROWSER=chrome #*** Uncomment and set vault base url and access token if you want to use vault to manage _CREDS secrets ***# #CREDENTIAL_VAULT_BASE_URL= -#CREDENTIAL_VAULT_TOKEN= #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= From 3304dca851aa2c1bb13abf4caade99449460c254 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Tue, 30 Jul 2019 16:51:32 -0500 Subject: [PATCH 6/8] =?UTF-8?q?MQE-1647:=C2=A0=20read=20vault=20token=20fr?= =?UTF-8?q?om=20local=20file=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 84d7a44) --- .../Handlers/SecretStorage/VaultStorage.php | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index bb88b18c5..785db036b 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -34,7 +34,11 @@ class VaultStorage extends BaseStorage */ const CONFIG_PATH_ENV_VAR = 'VAULT_CONFIG_PATH'; - const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(.+)$~"; + /** + * Regex to grab token helper script + */ + const TOKEN_HELPER_REGEX_GROUP_NAME = 'GROUP_NAME'; + const TOKEN_HELPER_REGEX = "~\s*token_helper\s*=(?<" . self::TOKEN_HELPER_REGEX_GROUP_NAME . ">.+)$~"; /** * Vault client @@ -146,35 +150,40 @@ private function readVaultTokenFromFileSystem() // Find user home directory $homeDir = getenv('HOME'); if ($homeDir === false) { - // If HOME is not set, don't fail right away - $homeDir = '~/'; - } else { - $homeDir = rtrim($homeDir, '/') . '/'; + throw new TestFrameworkException( + "HOME environment variable is not set. It's required when using vault." + ); } + $homeDir = realpath($homeDir) . DIRECTORY_SEPARATOR; + // Read .vault-token file if it is found in default location $vaultTokenFile = $homeDir . self::TOKEN_FILE; if (file_exists($vaultTokenFile)) { - // Found .vault-token file in default location, construct command - $cmd = 'cat ' . $vaultTokenFile; - } else { - // Otherwise search vault config file for custom token helper script - $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); - if ($vaultConfigPath === false) { - $vaultConfigFile = $homeDir . self::CONFIG_FILE; - } else { - $vaultConfigFile = rtrim($vaultConfigPath, '/') . '/' . self::CONFIG_FILE; + $token = file_get_contents($vaultTokenFile); + if ($token !== false) { + $this->token = $token; + return; } - // Found .vault config file, read custom token helper script and construct command - if (file_exists($vaultConfigFile) - && !empty($cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)))) { - $cmd = $cmd . ' get'; - } else { - throw new TestFrameworkException( - 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' - ); + } + + // Otherwise search vault config file for custom token helper script + $vaultConfigPath = getenv(self::CONFIG_PATH_ENV_VAR); + if ($vaultConfigPath === false) { + $vaultConfigFile = $homeDir . self::CONFIG_FILE; + } else { + $vaultConfigFile = realpath($vaultConfigPath) . DIRECTORY_SEPARATOR . self::CONFIG_FILE; + } + // Get custom token helper script file from .vault config file + if (file_exists($vaultConfigFile)) { + $cmd = $this->getTokenHelperScript(file($vaultConfigFile, FILE_IGNORE_NEW_LINES)); + if (!empty($cmd)) { + $this->token = $this->execVaultTokenHelper($cmd . ' get'); + return; } } - $this->token = $this->execVaultTokenHelper($cmd); + throw new TestFrameworkException( + 'Unable to read .vault-token file. Please authenticate to vault through vault CLI first.' + ); } /** @@ -188,8 +197,8 @@ private function getTokenHelperScript($lines) $tokenHelper = ''; foreach ($lines as $line) { preg_match(self::TOKEN_HELPER_REGEX, $line, $matches); - if (isset($matches[1])) { - $tokenHelper = trim(trim(trim($matches[1]), '"')); + if (isset($matches[self::TOKEN_HELPER_REGEX_GROUP_NAME])) { + $tokenHelper = trim(trim(trim($matches[self::TOKEN_HELPER_REGEX_GROUP_NAME]), '"')); } } return $tokenHelper; From f9baa07162a9b26fadbab194eb539bc373190bf7 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Wed, 7 Aug 2019 17:51:01 -0500 Subject: [PATCH 7/8] =?UTF-8?q?MQE-1647:=C2=A0=20read=20vault=20token=20fr?= =?UTF-8?q?om=20local=20file=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - allow secret base path configurable --- etc/config/.env.example | 5 ++-- .../Handlers/CredentialStore.php | 10 +++++--- .../Handlers/SecretStorage/VaultStorage.php | 25 +++++++++++++------ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/etc/config/.env.example b/etc/config/.env.example index f091688f5..369fd946b 100644 --- a/etc/config/.env.example +++ b/etc/config/.env.example @@ -30,8 +30,9 @@ BROWSER=chrome #MAGENTO_RESTAPI_SERVER_PORT=8080 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https -#*** Uncomment and set vault base url and access token if you want to use vault to manage _CREDS secrets ***# -#CREDENTIAL_VAULT_BASE_URL= +#*** Uncomment and set vault address and secret base path if you want to use vault to manage _CREDS secrets ***# +#CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200 +#CREDENTIAL_VAULT_SECRET_BASE_PATH=secret #*** Uncomment these properties to set up a dev environment with symlinked projects ***# #TESTS_BP= diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php index 75707b941..0514cfe57 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php @@ -58,10 +58,14 @@ private function __construct() } // Initialize vault storage - $csBaseUrl = getenv('CREDENTIAL_VAULT_BASE_URL'); - if ($csBaseUrl !== false) { + $cvAddress = getenv('CREDENTIAL_VAULT_ADDRESS'); + $cvSecretPath = getenv('CREDENTIAL_VAULT_SECRET_BASE_PATH'); + if ($cvAddress !== false && $cvSecretPath !== false) { try { - $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage(rtrim($csBaseUrl, '/')); + $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage( + rtrim($cvAddress, '/'), + '/' . trim($cvSecretPath, '/') + ); } catch (TestFrameworkException $e) { } } diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 785db036b..86e06b89a 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -14,12 +14,14 @@ class VaultStorage extends BaseStorage { + /** + * Mftf project path + */ const MFTF_PATH = '/mftf'; /** - * Adobe Vault + * Vault kv version 2 data */ - const BASE_PATH = '/dx_magento_qe'; - const KV_DATA = 'data'; + const KV2_DATA = 'data'; /** * Default vault token file @@ -54,18 +56,27 @@ class VaultStorage extends BaseStorage */ private $token = null; + /** + * Vault secret base path + * + * @var string + */ + private $secretBasePath; + /** * CredentialVault constructor * * @param string $baseUrl + * @param string $secretBasePath * @throws TestFrameworkException */ - public function __construct($baseUrl) + public function __construct($baseUrl, $secretBasePath) { parent::__construct(); if (null === $this->client) { // Creating the client using Guzzle6 Transport and passing a custom url $this->client = new Client(new Guzzle6Transport(['base_uri' => $baseUrl])); + $this->secretBasePath = $secretBasePath; } $this->readVaultTokenFromFileSystem(); if (!$this->authenticated()) { @@ -96,15 +107,15 @@ public function getEncryptedValue($key) try { // Split vendor/key to construct secret path list($vendor, $key) = explode('/', trim($key, '/'), 2); - $url = self::BASE_PATH - . (empty(self::KV_DATA) ? '' : '/' . self::KV_DATA) + $url = $this->secretBasePath + . (empty(self::KV2_DATA) ? '' : '/' . self::KV2_DATA) . self::MFTF_PATH . '/' . $vendor . '/' . $key; // Read value by key from vault - $value = $this->client->read($url)->getData()[self::KV_DATA][$key]; + $value = $this->client->read($url)->getData()[self::KV2_DATA][$key]; // Encrypt value for return $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv); parent::$cachedSecretData[$key] = $reValue; From 81dc8eccbc6e5bede51da82671ec4a9288e7a689 Mon Sep 17 00:00:00 2001 From: Ji Lu Date: Thu, 8 Aug 2019 09:48:41 -0500 Subject: [PATCH 8/8] =?UTF-8?q?MQE-1647:=C2=A0=20read=20vault=20token=20fr?= =?UTF-8?q?om=20local=20file=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - allow secret base path configurable --- .../DataGenerator/Handlers/SecretStorage/VaultStorage.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php index 86e06b89a..6a9e9f0cf 100644 --- a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php +++ b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/VaultStorage.php @@ -18,6 +18,7 @@ class VaultStorage extends BaseStorage * Mftf project path */ const MFTF_PATH = '/mftf'; + /** * Vault kv version 2 data */ @@ -27,10 +28,12 @@ class VaultStorage extends BaseStorage * Default vault token file */ const TOKEN_FILE = '.vault-token'; + /** * Default vault config file */ const CONFIG_FILE = '.vault'; + /** * Environment variable name for vault config path */