From d305b4c44a87d433869ba804b3857b44bca054d7 Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Thu, 18 Jul 2019 13:46:57 -0500 Subject: [PATCH 1/4] MQE-1606: Mftf static check script should return error exit code when check fails --- .../Console/StaticChecksCommand.php | 10 ++++++++ .../StaticCheck/StaticCheckInterface.php | 6 +++++ .../StaticCheck/TestDependencyCheck.php | 25 +++++++++++++++---- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php b/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php index fde306791..6b8510695 100644 --- a/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php @@ -47,10 +47,20 @@ protected function configure() protected function execute(InputInterface $input, OutputInterface $output) { $staticCheckObjects = $this->staticChecksList->getStaticChecks(); + + $errors = []; + foreach ($staticCheckObjects as $staticCheck) { $staticOutput = $staticCheck->execute($input); LoggingUtil::getInstance()->getLogger(get_class($staticCheck))->info($staticOutput); $output->writeln($staticOutput); + $errors += $staticCheck->getErrors(); + } + + if (empty($errors)) { + return 0; + } else { + return 1; } } } diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php index 45ebdea8d..da0bc42de 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php @@ -19,4 +19,10 @@ interface StaticCheckInterface * @return string */ public function execute(InputInterface $input); + + /** + * Return array containing all errors found after running the execute() function. + * @return array + */ + public function getErrors(); } diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php index 1a149a657..e87773c05 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php @@ -62,6 +62,12 @@ class TestDependencyCheck implements StaticCheckInterface */ private $alreadyExtractedDependencies; + /** + * Array containing all errors found after running the execute() function. + * @var array + */ + private $errors; + /** * Checks test dependencies, determined by references in tests versus the dependencies listed in the Magento module * @@ -98,13 +104,22 @@ public function execute(InputInterface $input) $actionGroupXmlFiles = $this->buildFileList($allModules, $filePaths[1]); $dataXmlFiles= $this->buildFileList($allModules, $filePaths[2]); - $testErrors = []; - $testErrors += $this->findErrorsInFileSet($testXmlFiles); - $testErrors += $this->findErrorsInFileSet($actionGroupXmlFiles); - $testErrors += $this->findErrorsInFileSet($dataXmlFiles); + $this->errors = []; + $this->errors += $this->findErrorsInFileSet($testXmlFiles); + $this->errors += $this->findErrorsInFileSet($actionGroupXmlFiles); + $this->errors += $this->findErrorsInFileSet($dataXmlFiles); //print all errors to file - return $this->printErrorsToFile($testErrors); + return $this->printErrorsToFile($this->getErrors()); + } + + /** + * Return array containing all errors found after running the execute() function. + * @return array + */ + public function getErrors() + { + return $this->errors; } /** From dfc77135cd773f5b16d4ab872dc6a8a11324f576 Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Thu, 18 Jul 2019 13:58:45 -0500 Subject: [PATCH 2/4] MQE-1606: Mftf static check script should return error exit code when check fails - Add to .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 63ea3d26e..d6ad9e81c 100755 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,6 @@ dev/tests/functional/MFTF.suite.yml dev/tests/functional/_output dev/mftf.log dev/tests/mftf.log -dev/tests/docs/* \ No newline at end of file +dev/tests/docs/* +dev/tests/_output +dev/tests/functional.suite.yml From c598015e35f603aab9c1adbb7873561512ee9bbc Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Mon, 22 Jul 2019 13:28:01 -0500 Subject: [PATCH 3/4] MQE-1606: Mftf static check script should return error exit code when check fails - Refactoring from code review --- .../Console/StaticChecksCommand.php | 13 ++++--- .../StaticCheck/StaticCheckInterface.php | 8 +++- .../StaticCheck/StaticCheckListInterface.php | 2 +- .../StaticCheck/StaticChecksList.php | 4 +- .../StaticCheck/TestDependencyCheck.php | 38 +++++++++++++------ 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php b/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php index 6b8510695..9757f494e 100644 --- a/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php +++ b/src/Magento/FunctionalTestingFramework/Console/StaticChecksCommand.php @@ -9,18 +9,19 @@ namespace Magento\FunctionalTestingFramework\Console; use Magento\FunctionalTestingFramework\StaticCheck\StaticChecksList; +use Magento\FunctionalTestingFramework\StaticCheck\StaticCheckListInterface; use Magento\FunctionalTestingFramework\Util\Logger\LoggingUtil; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; -use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Output\OutputInterface; +use Exception; class StaticChecksCommand extends Command { /** * Pool of static check scripts to run * - * @var \Magento\FunctionalTestingFramework\StaticCheck\StaticCheckListInterface + * @var StaticCheckListInterface */ private $staticChecksList; @@ -41,8 +42,8 @@ protected function configure() * * @param InputInterface $input * @param OutputInterface $output - * @return int|null|void - * @throws \Exception + * @return int + * @throws Exception */ protected function execute(InputInterface $input, OutputInterface $output) { @@ -51,7 +52,9 @@ protected function execute(InputInterface $input, OutputInterface $output) $errors = []; foreach ($staticCheckObjects as $staticCheck) { - $staticOutput = $staticCheck->execute($input); + $staticCheck->execute($input); + + $staticOutput = $staticCheck->getOutput(); LoggingUtil::getInstance()->getLogger(get_class($staticCheck))->info($staticOutput); $output->writeln($staticOutput); $errors += $staticCheck->getErrors(); diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php index da0bc42de..54bdd3baf 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckInterface.php @@ -16,7 +16,7 @@ interface StaticCheckInterface /** * Executes static check script, returns output. * @param InputInterface $input - * @return string + * @return void */ public function execute(InputInterface $input); @@ -25,4 +25,10 @@ public function execute(InputInterface $input); * @return array */ public function getErrors(); + + /** + * Return string of a short human readable result of the check. For example: "No Dependency errors found." + * @return string + */ + public function getOutput(); } diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckListInterface.php b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckListInterface.php index a8f1a0f6c..867c29e71 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckListInterface.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticCheckListInterface.php @@ -14,7 +14,7 @@ interface StaticCheckListInterface /** * Gets list of static check script instances * - * @return \Magento\FunctionalTestingFramework\StaticCheck\StaticCheckListInterface[] + * @return StaticCheckInterface[] */ public function getStaticChecks(); } diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php index d66d941c6..f3cf20739 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/StaticChecksList.php @@ -16,14 +16,14 @@ class StaticChecksList implements StaticCheckListInterface /** * Property contains all static check scripts. * - * @var \Magento\FunctionalTestingFramework\StaticCheck\StaticCheckListInterface[] + * @var StaticCheckInterface[] */ private $checks; /** * Constructor * - * @param array $scripts + * @param array $checks */ public function __construct(array $checks = []) { diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php index e87773c05..35f4a5ccf 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php @@ -6,20 +6,20 @@ namespace Magento\FunctionalTestingFramework\StaticCheck; -use Magento\FunctionalTestingFramework\Config\Data; use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig; use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler; +use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException; +use Magento\FunctionalTestingFramework\Exceptions\XmlException; use Magento\FunctionalTestingFramework\Page\Handlers\PageObjectHandler; use Magento\FunctionalTestingFramework\Page\Handlers\SectionObjectHandler; -use Magento\FunctionalTestingFramework\Page\Objects\SectionObject; use Magento\FunctionalTestingFramework\Test\Handlers\ActionGroupObjectHandler; use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler; use Magento\FunctionalTestingFramework\Test\Objects\ActionObject; use Magento\FunctionalTestingFramework\Util\ModuleResolver; use Magento\FunctionalTestingFramework\Util\TestGenerator; use Symfony\Component\Console\Input\InputInterface; -use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\Finder\Finder; +use Exception; /** * Class TestDependencyCheck @@ -68,11 +68,18 @@ class TestDependencyCheck implements StaticCheckInterface */ private $errors; + /** + * String representing the output summary found after running the execute() function. + * @var string + */ + private $output; + /** * Checks test dependencies, determined by references in tests versus the dependencies listed in the Magento module * * @param InputInterface $input * @return string + * @throws Exception; * @SuppressWarnings(PHPMD.CyclomaticComplexity) */ public function execute(InputInterface $input) @@ -109,8 +116,8 @@ public function execute(InputInterface $input) $this->errors += $this->findErrorsInFileSet($actionGroupXmlFiles); $this->errors += $this->findErrorsInFileSet($dataXmlFiles); - //print all errors to file - return $this->printErrorsToFile($this->getErrors()); + // hold on to the output and print any errors to a file + $this->output = $this->printErrorsToFile(); } /** @@ -122,12 +129,17 @@ public function getErrors() return $this->errors; } + public function getOutput() + { + return $this->output; + } + /** * Finds all reference errors in given set of files * @param Finder $files * @return array - * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException - * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException + * @throws TestReferenceException + * @throws XmlException */ private function findErrorsInFileSet($files) { @@ -348,8 +360,7 @@ private function buildFileList($modulePaths, $path) * Attempts to find any MFTF entity by its name. Returns null if none are found. * @param string $name * @return mixed - * @throws \Magento\FunctionalTestingFramework\Exceptions\TestReferenceException - * @throws \Magento\FunctionalTestingFramework\Exceptions\XmlException + * @throws XmlException */ private function findEntity($name) { @@ -378,24 +389,29 @@ private function findEntity($name) /** * Prints out given errors to file, and returns summary result string - * @param array $errors * @return string */ - private function printErrorsToFile($errors) + private function printErrorsToFile() { + $errors = $this->getErrors(); + if (empty($errors)) { return "No Dependency errors found."; } + $outputPath = getcwd() . DIRECTORY_SEPARATOR . "mftf-dependency-checks.txt"; $fileResource = fopen($outputPath, 'w'); $header = "MFTF File Dependency Check:\n"; fwrite($fileResource, $header); + foreach ($errors as $test => $error) { fwrite($fileResource, $error[0] . PHP_EOL); } + fclose($fileResource); $errorCount = count($errors); $output = "Dependency errors found across {$errorCount} file(s). Error details output to {$outputPath}"; + return $output; } } From 15ea947d517b315edc14ef7b5418f19b4c6f8741 Mon Sep 17 00:00:00 2001 From: Tom Reece Date: Mon, 22 Jul 2019 13:42:52 -0500 Subject: [PATCH 4/4] MQE-1606: Mftf static check script should return error exit code when check fails - Add missing phpdoc --- .../StaticCheck/TestDependencyCheck.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php index 35f4a5ccf..a891d42e1 100644 --- a/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php +++ b/src/Magento/FunctionalTestingFramework/StaticCheck/TestDependencyCheck.php @@ -129,6 +129,10 @@ public function getErrors() return $this->errors; } + /** + * Return string of a short human readable result of the check. For example: "No Dependency errors found." + * @return string + */ public function getOutput() { return $this->output;