From 06fba656f2d26335711b75b961a0f81868d4f293 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Thu, 29 Oct 2020 21:50:11 +0100 Subject: [PATCH 01/23] Implementing variables extraction in "translation:update" --- .../NodeVisitor/TranslationNodeVisitor.php | 69 +++++++++++++++++-- ...ranslationDefaultDomainNodeVisitorTest.php | 4 +- .../TranslationNodeVisitorTest.php | 10 +-- .../Bridge/Twig/Translation/TwigExtractor.php | 13 +++- .../Command/TranslationUpdateCommand.php | 8 +++ .../Translation/Catalogue/MergeOperation.php | 14 ++-- .../Translation/Catalogue/TargetOperation.php | 14 ++-- .../Translation/Extractor/PhpExtractor.php | 66 +++++++++++++++--- .../Tests/Catalogue/MergeOperationTest.php | 13 ++++ .../Tests/Catalogue/TargetOperationTest.php | 2 +- 10 files changed, 173 insertions(+), 40 deletions(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index e5b0fc4ea1b73..bb4a6a6662bd5 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -30,6 +30,24 @@ final class TranslationNodeVisitor extends AbstractNodeVisitor public const UNDEFINED_DOMAIN = '_undefined'; private $enabled = false; + /** + * This array stores found messages. + * + * The data structure of this array is as follows: + * + * [ + * 0 => [ + * 0 => 'message', + * 1 => 'domain', + * 2 => [ + * 'variable1', + * 'variable2', + * ... + * ] + * ], + * ... + * ] + */ private $messages = []; public function enable(): void @@ -67,6 +85,7 @@ protected function doEnterNode(Node $node, Environment $env): Node $this->messages[] = [ $node->getNode('node')->getAttribute('value'), $this->getReadDomainFromArguments($node->getNode('arguments'), 1), + $this->getReadVariablesFromArguments($node->getNode('arguments'), 0), ]; } elseif ( $node instanceof FilterExpression && @@ -74,12 +93,20 @@ protected function doEnterNode(Node $node, Environment $env): Node $node->getNode('node') instanceof FunctionExpression && 't' === $node->getNode('node')->getAttribute('name') ) { - $nodeArguments = $node->getNode('node')->getNode('arguments'); + // extract t() nodes with a trans filter applied + $filterNodeArguments = $node->getNode('arguments'); + $functionNodeArguments = $node->getNode('node')->getNode('arguments'); + + if ($functionNodeArguments->getIterator()->current() instanceof ConstantExpression) { + // Get domain from filter (if available) this will support also "trans_default_domain" + // but fallback to the function argument to support the following: + // {{ t("new key", {}, "domain") | trans() }} + $domain = $this->getReadDomainFromArguments($filterNodeArguments, 1) ?? $this->getReadDomainFromArguments($functionNodeArguments, 2); - if ($nodeArguments->getIterator()->current() instanceof ConstantExpression) { $this->messages[] = [ - $this->getReadMessageFromArguments($nodeArguments, 0), - $this->getReadDomainFromArguments($nodeArguments, 2), + $this->getReadMessageFromArguments($functionNodeArguments, 0), + $domain, + $this->getReadVariablesFromArguments($functionNodeArguments, 1), ]; } } elseif ($node instanceof TransNode) { @@ -87,6 +114,7 @@ protected function doEnterNode(Node $node, Environment $env): Node $this->messages[] = [ $node->getNode('body')->getAttribute('data'), $node->hasNode('domain') ? $this->getReadDomainFromNode($node->getNode('domain')) : null, + $this->getReadVariablesFromArguments($node, 0), ]; } elseif ( $node instanceof FilterExpression && @@ -141,6 +169,39 @@ private function getReadMessageFromNode(Node $node): ?string return null; } + private function getReadVariablesFromArguments(Node $arguments, int $index): array + { + if ($arguments->hasNode('vars')) { + $argument = $arguments->getNode('vars'); + } elseif ($arguments->hasNode($index)) { + $argument = $arguments->getNode($index); + } else { + return []; + } + + return $this->getReadVariablesFromNode($argument); + } + + private function getReadVariablesFromNode(Node $node): ?array + { + if (!empty($node)) { + $variables = []; + + foreach ($node as $key => $variable) { + // Odd children are variable names, even ones are values + if ($key % 2 == 1) { + continue; + } + + $variables[] = $variable->getAttribute('value'); + } + + return $variables; + } + + return []; + } + private function getReadDomainFromArguments(Node $arguments, int $index): ?string { if ($arguments->hasNode('domain')) { diff --git a/src/Symfony/Bridge/Twig/Tests/NodeVisitor/TranslationDefaultDomainNodeVisitorTest.php b/src/Symfony/Bridge/Twig/Tests/NodeVisitor/TranslationDefaultDomainNodeVisitorTest.php index cc2b6ef2ac39e..2311447c00e4f 100644 --- a/src/Symfony/Bridge/Twig/Tests/NodeVisitor/TranslationDefaultDomainNodeVisitorTest.php +++ b/src/Symfony/Bridge/Twig/Tests/NodeVisitor/TranslationDefaultDomainNodeVisitorTest.php @@ -47,7 +47,7 @@ public function testDefaultDomainAssignment(Node $node) $visitor->enterNode($node, $env); $visitor->leaveNode($node, $env); - $this->assertEquals([[self::$message, self::$domain]], $visitor->getMessages()); + $this->assertEquals([[self::$message, self::$domain, []]], $visitor->getMessages()); } /** @dataProvider getDefaultDomainAssignmentTestData */ @@ -73,7 +73,7 @@ public function testNewModuleWithoutDefaultDomainTag(Node $node) $visitor->enterNode($node, $env); $visitor->leaveNode($node, $env); - $this->assertEquals([[self::$message, null]], $visitor->getMessages()); + $this->assertEquals([[self::$message, null, []]], $visitor->getMessages()); } public function getDefaultDomainAssignmentTestData() diff --git a/src/Symfony/Bridge/Twig/Tests/NodeVisitor/TranslationNodeVisitorTest.php b/src/Symfony/Bridge/Twig/Tests/NodeVisitor/TranslationNodeVisitorTest.php index 069914a4fc066..0b254f4e77cc6 100644 --- a/src/Symfony/Bridge/Twig/Tests/NodeVisitor/TranslationNodeVisitorTest.php +++ b/src/Symfony/Bridge/Twig/Tests/NodeVisitor/TranslationNodeVisitorTest.php @@ -48,7 +48,7 @@ public function testMessageExtractionWithInvalidDomainNode() 0 ); - $this->testMessagesExtraction($node, [[$message, TranslationNodeVisitor::UNDEFINED_DOMAIN]]); + $this->testMessagesExtraction($node, [[$message, TranslationNodeVisitor::UNDEFINED_DOMAIN, []]]); } public function getMessagesExtractionTestData() @@ -57,10 +57,10 @@ public function getMessagesExtractionTestData() $domain = 'domain'; return [ - [TwigNodeProvider::getTransFilter($message), [[$message, null]]], - [TwigNodeProvider::getTransTag($message), [[$message, null]]], - [TwigNodeProvider::getTransFilter($message, $domain), [[$message, $domain]]], - [TwigNodeProvider::getTransTag($message, $domain), [[$message, $domain]]], + [TwigNodeProvider::getTransFilter($message), [[$message, null, []]]], + [TwigNodeProvider::getTransTag($message), [[$message, null, []]]], + [TwigNodeProvider::getTransFilter($message, $domain), [[$message, $domain, []]]], + [TwigNodeProvider::getTransTag($message, $domain), [[$message, $domain, []]]], ]; } } diff --git a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php index e79ec697e0f50..a65506feb8e35 100644 --- a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php +++ b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php @@ -78,7 +78,18 @@ protected function extractTemplate(string $template, MessageCatalogue $catalogue $this->twig->parse($this->twig->tokenize(new Source($template, ''))); foreach ($visitor->getMessages() as $message) { - $catalogue->set(trim($message[0]), $this->prefix.trim($message[0]), $message[1] ?: $this->defaultDomain); + $id = trim($message[0]); + + $catalogue->set($id, $this->prefix.trim($message[0]), $message[1] ?: $this->defaultDomain); + + if (!empty($message[2])) { + $catalogue->setMetadata($id, ['notes' => [ + [ + 'category' => 'symfony-extractor-variables', + 'content' => 'Available variables: ' . join(', ', $message[2]), + ] + ]], $message[1] ?: $this->defaultDomain); + } } $visitor->disable(); diff --git a/src/Symfony/Bundle/FrameworkBundle/Command/TranslationUpdateCommand.php b/src/Symfony/Bundle/FrameworkBundle/Command/TranslationUpdateCommand.php index cacf32845147d..03d86191b07b5 100644 --- a/src/Symfony/Bundle/FrameworkBundle/Command/TranslationUpdateCommand.php +++ b/src/Symfony/Bundle/FrameworkBundle/Command/TranslationUpdateCommand.php @@ -240,6 +240,14 @@ protected function execute(InputInterface $input, OutputInterface $output): int $currentMessages = array_diff_key($newMessages, $result->all($domain)); $result->replace($currentMessages, $domain); $result->replace($allIntlMessages + $newMessages, $intlDomain); + + // Move new metadata + foreach ($newMessages as $key => $message) { + if (null !== $extractedCatalogue->getMetadata($key, $domain)) { + $result->setMetadata($key, $extractedCatalogue->getMetadata($key, $domain), $intlDomain); + $extractedCatalogue->deleteMetadata($key, $domain); + } + } } } diff --git a/src/Symfony/Component/Translation/Catalogue/MergeOperation.php b/src/Symfony/Component/Translation/Catalogue/MergeOperation.php index 87db2fb031d96..e10b68c4226df 100644 --- a/src/Symfony/Component/Translation/Catalogue/MergeOperation.php +++ b/src/Symfony/Component/Translation/Catalogue/MergeOperation.php @@ -38,10 +38,9 @@ protected function processDomain(string $domain) foreach ($this->source->all($domain) as $id => $message) { $this->messages[$domain]['all'][$id] = $message; - $d = $this->source->defines($id, $intlDomain) ? $intlDomain : $domain; - $this->result->add([$id => $message], $d); - if (null !== $keyMetadata = $this->source->getMetadata($id, $d)) { - $this->result->setMetadata($id, $keyMetadata, $d); + $this->result->add([$id => $message], $this->source->defines($id, $intlDomain) ? $intlDomain : $domain); + if (null !== $keyMetadata = $this->source->getMetadata($id, $this->source->defines($id, $intlDomain) ? $intlDomain : $domain)) { + $this->result->setMetadata($id, $keyMetadata, $this->source->defines($id, $intlDomain) ? $intlDomain : $domain); } } @@ -49,10 +48,9 @@ protected function processDomain(string $domain) if (!$this->source->has($id, $domain)) { $this->messages[$domain]['all'][$id] = $message; $this->messages[$domain]['new'][$id] = $message; - $d = $this->target->defines($id, $intlDomain) ? $intlDomain : $domain; - $this->result->add([$id => $message], $d); - if (null !== $keyMetadata = $this->target->getMetadata($id, $d)) { - $this->result->setMetadata($id, $keyMetadata, $d); + $this->result->add([$id => $message], $this->target->defines($id, $intlDomain) ? $intlDomain : $domain); + if (null !== $keyMetadata = $this->target->getMetadata($id, $this->target->defines($id, $intlDomain) ? $intlDomain : $domain)) { + $this->result->setMetadata($id, $keyMetadata, $this->target->defines($id, $intlDomain) ? $intlDomain : $domain); } } } diff --git a/src/Symfony/Component/Translation/Catalogue/TargetOperation.php b/src/Symfony/Component/Translation/Catalogue/TargetOperation.php index 399d917409399..a17c05748b54d 100644 --- a/src/Symfony/Component/Translation/Catalogue/TargetOperation.php +++ b/src/Symfony/Component/Translation/Catalogue/TargetOperation.php @@ -49,10 +49,9 @@ protected function processDomain(string $domain) foreach ($this->source->all($domain) as $id => $message) { if ($this->target->has($id, $domain)) { $this->messages[$domain]['all'][$id] = $message; - $d = $this->target->defines($id, $intlDomain) ? $intlDomain : $domain; - $this->result->add([$id => $message], $d); - if (null !== $keyMetadata = $this->source->getMetadata($id, $d)) { - $this->result->setMetadata($id, $keyMetadata, $d); + $this->result->add([$id => $message], $this->source->defines($id, $intlDomain) ? $intlDomain : $domain); + if (null !== $keyMetadata = $this->source->getMetadata($id, $this->source->defines($id, $intlDomain) ? $intlDomain : $domain)) { + $this->result->setMetadata($id, $keyMetadata, $this->source->defines($id, $intlDomain) ? $intlDomain : $domain); } } else { $this->messages[$domain]['obsolete'][$id] = $message; @@ -63,10 +62,9 @@ protected function processDomain(string $domain) if (!$this->source->has($id, $domain)) { $this->messages[$domain]['all'][$id] = $message; $this->messages[$domain]['new'][$id] = $message; - $d = $this->target->defines($id, $intlDomain) ? $intlDomain : $domain; - $this->result->add([$id => $message], $d); - if (null !== $keyMetadata = $this->target->getMetadata($id, $d)) { - $this->result->setMetadata($id, $keyMetadata, $d); + $this->result->add([$id => $message], $this->target->defines($id, $intlDomain) ? $intlDomain : $domain); + if (null !== $keyMetadata = $this->target->getMetadata($id, $this->target->defines($id, $intlDomain) ? $intlDomain : $domain)) { + $this->result->setMetadata($id, $keyMetadata, $this->target->defines($id, $intlDomain) ? $intlDomain : $domain); } } } diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index c5efb5f3b5b4b..a4b0b9c437a0f 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -182,25 +182,58 @@ private function seekToNextRelevantToken(\Iterator $tokenIterator) } } - private function skipMethodArgument(\Iterator $tokenIterator) + /** + * @return string[] + */ + private function getArrayKeys(\Iterator $tokenIterator) { - $openBraces = 0; + $keys = []; - for (; $tokenIterator->valid(); $tokenIterator->next()) { + // Skip opening "[" + $tokenIterator->next(); + + while ($tokenIterator->valid()) { + $this->seekToNextRelevantToken($tokenIterator); $t = $tokenIterator->current(); - if ('[' === $t[0] || '(' === $t[0]) { - ++$openBraces; - } + // End of main array + if (']' === $t[0]) { + // Skip following "," + $tokenIterator->next(); - if (']' === $t[0] || ')' === $t[0]) { - --$openBraces; + return $keys; } - if ((0 === $openBraces && ',' === $t[0]) || (-1 === $openBraces && ')' === $t[0])) { - break; + // Get array key + $keys[] = $this->getValue($tokenIterator); + + // Skip value + $openBraces = 0; + for (; $tokenIterator->valid(); $tokenIterator->next()) { + $t = $tokenIterator->current(); + + if ('[' === $t[0] || '(' === $t[0]) { + ++$openBraces; + } + + if (']' === $t[0] || ')' === $t[0]) { + --$openBraces; + } + + if (0 === $openBraces && ',' === $t[0]) { + // Skip following "," + $tokenIterator->next(); + + break; + } + + if (-1 === $openBraces && ']' === $t[0]) { + break; + } } } + + return $keys; } /** @@ -274,6 +307,7 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string foreach ($this->sequences as $sequence) { $message = ''; $domain = 'messages'; + $variables = []; $tokenIterator->seek($key); foreach ($sequence as $sequenceKey => $item) { @@ -289,7 +323,7 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string break; } } elseif (self::METHOD_ARGUMENTS_TOKEN === $item) { - $this->skipMethodArgument($tokenIterator); + $variables = $this->getArrayKeys($tokenIterator); } elseif (self::DOMAIN_TOKEN === $item) { $domainToken = $this->getValue($tokenIterator); if ('' !== $domainToken) { @@ -308,6 +342,16 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string $normalizedFilename = preg_replace('{[\\\\/]+}', '/', $filename); $metadata['sources'][] = $normalizedFilename.':'.$tokens[$key][2]; $catalog->setMetadata($message, $metadata, $domain); + + if (!empty($variables)) { + $catalog->setMetadata($message, ['notes' => [ + [ + 'category' => 'symfony-extractor-variables', + 'content' => 'Available variables: ' . join(', ', $variables), + ] + ]], $domain); + } + break; } } diff --git a/src/Symfony/Component/Translation/Tests/Catalogue/MergeOperationTest.php b/src/Symfony/Component/Translation/Tests/Catalogue/MergeOperationTest.php index 240c492800acc..f14b367b6dcf9 100644 --- a/src/Symfony/Component/Translation/Tests/Catalogue/MergeOperationTest.php +++ b/src/Symfony/Component/Translation/Tests/Catalogue/MergeOperationTest.php @@ -67,6 +67,19 @@ public function testGetResultFromIntlDomain() ); } + public function testGetResultWithMixedDomains() + { + $this->assertEquals( + new MessageCatalogue('en', [ + 'messages' => ['a' => 'old_a'], + ]), + $this->createOperation( + new MessageCatalogue('en', ['messages' => ['a' => 'old_a']]), + new MessageCatalogue('en', ['messages+intl-icu' => ['a' => 'new_a']]) + )->getResult() + ); + } + public function testGetResultWithMetadata() { $leftCatalogue = new MessageCatalogue('en', ['messages' => ['a' => 'old_a', 'b' => 'old_b']]); diff --git a/src/Symfony/Component/Translation/Tests/Catalogue/TargetOperationTest.php b/src/Symfony/Component/Translation/Tests/Catalogue/TargetOperationTest.php index 354c213e9e3b7..fbd80c79dcb68 100644 --- a/src/Symfony/Component/Translation/Tests/Catalogue/TargetOperationTest.php +++ b/src/Symfony/Component/Translation/Tests/Catalogue/TargetOperationTest.php @@ -71,7 +71,7 @@ public function testGetResultWithMixedDomains() { $this->assertEquals( new MessageCatalogue('en', [ - 'messages+intl-icu' => ['a' => 'old_a'], + 'messages' => ['a' => 'old_a'], ]), $this->createOperation( new MessageCatalogue('en', ['messages' => ['a' => 'old_a']]), From e2934b599a84de5de300bdb54951665942dc2c3c Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Thu, 29 Oct 2020 23:02:16 +0100 Subject: [PATCH 02/23] Applying patch from fabbot --- .../Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php | 2 +- src/Symfony/Bridge/Twig/Translation/TwigExtractor.php | 4 ++-- src/Symfony/Component/Translation/Extractor/PhpExtractor.php | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index bb4a6a6662bd5..37881cd4e1f65 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -189,7 +189,7 @@ private function getReadVariablesFromNode(Node $node): ?array foreach ($node as $key => $variable) { // Odd children are variable names, even ones are values - if ($key % 2 == 1) { + if (1 == $key % 2) { continue; } diff --git a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php index a65506feb8e35..6c8c6a01d6f1a 100644 --- a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php +++ b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php @@ -86,8 +86,8 @@ protected function extractTemplate(string $template, MessageCatalogue $catalogue $catalogue->setMetadata($id, ['notes' => [ [ 'category' => 'symfony-extractor-variables', - 'content' => 'Available variables: ' . join(', ', $message[2]), - ] + 'content' => 'Available variables: '.implode(', ', $message[2]), + ], ]], $message[1] ?: $this->defaultDomain); } } diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index a4b0b9c437a0f..248a4ffc3595e 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -347,8 +347,8 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string $catalog->setMetadata($message, ['notes' => [ [ 'category' => 'symfony-extractor-variables', - 'content' => 'Available variables: ' . join(', ', $variables), - ] + 'content' => 'Available variables: '.implode(', ', $variables), + ], ]], $domain); } From bedb0be26799939a8f0249421be3e1b4090a5b69 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Fri, 30 Oct 2020 03:57:05 +0100 Subject: [PATCH 03/23] Fixing regression: Twig t() now works when domain is set with trans_default_domain --- .../Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index 37881cd4e1f65..c99fd161b4bce 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -94,18 +94,12 @@ protected function doEnterNode(Node $node, Environment $env): Node 't' === $node->getNode('node')->getAttribute('name') ) { // extract t() nodes with a trans filter applied - $filterNodeArguments = $node->getNode('arguments'); $functionNodeArguments = $node->getNode('node')->getNode('arguments'); if ($functionNodeArguments->getIterator()->current() instanceof ConstantExpression) { - // Get domain from filter (if available) this will support also "trans_default_domain" - // but fallback to the function argument to support the following: - // {{ t("new key", {}, "domain") | trans() }} - $domain = $this->getReadDomainFromArguments($filterNodeArguments, 1) ?? $this->getReadDomainFromArguments($functionNodeArguments, 2); - $this->messages[] = [ $this->getReadMessageFromArguments($functionNodeArguments, 0), - $domain, + $this->getReadDomainFromArguments($functionNodeArguments, 2), $this->getReadVariablesFromArguments($functionNodeArguments, 1), ]; } From 87d542f9d5363d9b8bd434b2c609040eeb205873 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Fri, 30 Oct 2020 17:08:08 +0100 Subject: [PATCH 04/23] Fixing variables metadata overwriting previously set data (PHP extractor) --- .../Component/Translation/Extractor/PhpExtractor.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index 248a4ffc3595e..19bffb2409f64 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -341,17 +341,16 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string $metadata = $catalog->getMetadata($message, $domain) ?? []; $normalizedFilename = preg_replace('{[\\\\/]+}', '/', $filename); $metadata['sources'][] = $normalizedFilename.':'.$tokens[$key][2]; - $catalog->setMetadata($message, $metadata, $domain); if (!empty($variables)) { - $catalog->setMetadata($message, ['notes' => [ - [ + $metadata['notes'][] = [ 'category' => 'symfony-extractor-variables', 'content' => 'Available variables: '.implode(', ', $variables), - ], - ]], $domain); + ]; } + $catalog->setMetadata($message, $metadata, $domain); + break; } } From 8142f75e2f9e6730d85b0969c8e44785e74b2871 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Fri, 30 Oct 2020 17:10:27 +0100 Subject: [PATCH 05/23] Updating existing tests to include variables --- .../Tests/Extractor/PhpExtractorTest.php | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php index 82ddfe0e8aeac..f399814f90d65 100644 --- a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php +++ b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php @@ -11,6 +11,7 @@ namespace Symfony\Component\Translation\Tests\Extractor; +use PHPUnit\Framework\ExpectationFailedException; use PHPUnit\Framework\TestCase; use Symfony\Component\Translation\Extractor\PhpExtractor; use Symfony\Component\Translation\MessageCatalogue; @@ -109,6 +110,78 @@ public function testExtraction($resource) 'typecast' => 'prefixtypecast', ], ]; + + // Expected metadata (variables) + $expectedVariables = [ + 'messages' => [ + 'translatable single-quoted key' => null, + 'translatable double-quoted key' => null, + 'translatable heredoc key' => null, + 'translatable nowdoc key' => null, + "translatable double-quoted key with whitespace and escaped \$\n\" sequences" => null, + 'translatable single-quoted key with whitespace and nonescaped \$\n\' sequences' => null, + 'translatable single-quoted key with "quote mark at the end"' => null, + 'translatable '.$expectedHeredoc => null, + 'translatable '.$expectedNowdoc => null, + 'translatable concatenated message with heredoc and nowdoc' => null, + 'translatable default domain' => null, + 'translatable-fqn single-quoted key' => null, + 'translatable-fqn double-quoted key' => null, + 'translatable-fqn heredoc key' => null, + 'translatable-fqn nowdoc key' => null, + "translatable-fqn double-quoted key with whitespace and escaped \$\n\" sequences" => null, + 'translatable-fqn single-quoted key with whitespace and nonescaped \$\n\' sequences' => null, + 'translatable-fqn single-quoted key with "quote mark at the end"' => null, + 'translatable-fqn '.$expectedHeredoc => null, + 'translatable-fqn '.$expectedNowdoc => null, + 'translatable-fqn concatenated message with heredoc and nowdoc' => null, + 'translatable-fqn default domain' => null, + 'translatable-short single-quoted key' => null, + 'translatable-short double-quoted key' => null, + 'translatable-short heredoc key' => null, + 'translatable-short nowdoc key' => null, + "translatable-short double-quoted key with whitespace and escaped \$\n\" sequences" => null, + 'translatable-short single-quoted key with whitespace and nonescaped \$\n\' sequences' => null, + 'translatable-short single-quoted key with "quote mark at the end"' => null, + 'translatable-short '.$expectedHeredoc => null, + 'translatable-short '.$expectedNowdoc => null, + 'translatable-short concatenated message with heredoc and nowdoc' => null, + 'translatable-short default domain' => null, + 'single-quoted key' => null, + 'double-quoted key' => null, + 'heredoc key' => null, + 'nowdoc key' => null, + "double-quoted key with whitespace and escaped \$\n\" sequences" => null, + 'single-quoted key with whitespace and nonescaped \$\n\' sequences' => null, + 'single-quoted key with "quote mark at the end"' => null, + $expectedHeredoc => null, + $expectedNowdoc => null, + 'concatenated message with heredoc and nowdoc' => null, + 'default domain' => null, + ], + 'not_messages' => [ + 'translatable other-domain-test-no-params-short-array' => null, + 'translatable other-domain-test-no-params-long-array' => null, + 'translatable other-domain-test-params-short-array' => 'Available variables: foo', + 'translatable other-domain-test-params-long-array' => 'Available variables: foo', + 'translatable typecast' => 'Available variables: a', + 'translatable-fqn other-domain-test-no-params-short-array' => null, + 'translatable-fqn other-domain-test-no-params-long-array' => null, + 'translatable-fqn other-domain-test-params-short-array' => 'Available variables: foo', + 'translatable-fqn other-domain-test-params-long-array' => 'Available variables: foo', + 'translatable-fqn typecast' => 'Available variables: a', + 'translatable-short other-domain-test-no-params-short-array' => null, + 'translatable-short other-domain-test-no-params-long-array' => null, + 'translatable-short other-domain-test-params-short-array' => 'Available variables: foo', + 'translatable-short other-domain-test-params-long-array' => 'Available variables: foo', + 'translatable-short typecast' => 'Available variables: a', + 'other-domain-test-no-params-short-array' => null, + 'other-domain-test-no-params-long-array' => null, + 'other-domain-test-params-short-array' => 'Available variables: foo', + 'other-domain-test-params-long-array' => 'Available variables: foo', + 'typecast' => 'Available variables: a', + ], + ]; $actualCatalogue = $catalogue->all(); $this->assertEquals($expectedCatalogue, $actualCatalogue); @@ -128,6 +201,17 @@ public function testExtraction($resource) $filename = str_replace(\DIRECTORY_SEPARATOR, '/', __DIR__).'/../fixtures/extractor/translation.html.php'; $this->assertEquals(['sources' => [$filename.':2']], $catalogue->getMetadata('single-quoted key')); $this->assertEquals(['sources' => [$filename.':37']], $catalogue->getMetadata('other-domain-test-no-params-short-array', 'not_messages')); + + // Check variables metadata + foreach (array_keys($actualCatalogue) as $domain) {echo "[{$domain}]\n"; + foreach (array_keys($actualCatalogue[$domain]) as $id) { + $this->assertTrue(array_key_exists($id, $expectedVariables[$domain]), 'Metadata for domain "'.$domain.'" and id "'.$id.'" was not expected!'); + + $extractedVariables = $this->getVariablesNoteContentFromMetadata($catalogue->getMetadata($id, $domain)); + + $this->assertEquals($expectedVariables[$domain][$id], $extractedVariables); + } + } } /** @@ -175,4 +259,21 @@ public function resourcesProvider() [new \ArrayObject($splFiles)], ]; } + + private function getVariablesNoteContentFromMetadata(array $metadata) + { + /** + * $metadata = [ + * 'notes' => [ + * 0 => [ + * 'category' => 'symfony-extractor-variables', + * 'content' => 'Available variables: var1, var2', + * ], + * ... + * ], + * ... + * ] + */ + return array_filter($metadata['notes'] ?? [], function ($note) { return $note['category'] === 'symfony-extractor-variables'; })[0]['content'] ?? null; + } } From a333f13e1b578302ea3d68f53bbee51847b276be Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Fri, 30 Oct 2020 17:12:49 +0100 Subject: [PATCH 06/23] Applying patch from fabbot --- .../Translation/Tests/Extractor/PhpExtractorTest.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php index f399814f90d65..5b7ccaf7fde07 100644 --- a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php +++ b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php @@ -11,7 +11,6 @@ namespace Symfony\Component\Translation\Tests\Extractor; -use PHPUnit\Framework\ExpectationFailedException; use PHPUnit\Framework\TestCase; use Symfony\Component\Translation\Extractor\PhpExtractor; use Symfony\Component\Translation\MessageCatalogue; @@ -203,9 +202,9 @@ public function testExtraction($resource) $this->assertEquals(['sources' => [$filename.':37']], $catalogue->getMetadata('other-domain-test-no-params-short-array', 'not_messages')); // Check variables metadata - foreach (array_keys($actualCatalogue) as $domain) {echo "[{$domain}]\n"; + foreach (array_keys($actualCatalogue) as $domain) { foreach (array_keys($actualCatalogue[$domain]) as $id) { - $this->assertTrue(array_key_exists($id, $expectedVariables[$domain]), 'Metadata for domain "'.$domain.'" and id "'.$id.'" was not expected!'); + $this->assertArrayHasKey($id, $expectedVariables[$domain], 'Metadata for domain "'.$domain.'" and id "'.$id.'" was not expected!'); $extractedVariables = $this->getVariablesNoteContentFromMetadata($catalogue->getMetadata($id, $domain)); @@ -262,7 +261,7 @@ public function resourcesProvider() private function getVariablesNoteContentFromMetadata(array $metadata) { - /** + /* * $metadata = [ * 'notes' => [ * 0 => [ @@ -274,6 +273,6 @@ private function getVariablesNoteContentFromMetadata(array $metadata) * ... * ] */ - return array_filter($metadata['notes'] ?? [], function ($note) { return $note['category'] === 'symfony-extractor-variables'; })[0]['content'] ?? null; + return array_filter($metadata['notes'] ?? [], function ($note) { return 'symfony-extractor-variables' === $note['category']; })[0]['content'] ?? null; } } From 38d9e5588ba225a4295ea7ed9539682aedff1196 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Fri, 30 Oct 2020 18:47:49 +0100 Subject: [PATCH 07/23] Supporting both long and short array syntax (PHP extractor) --- .../Translation/Extractor/PhpExtractor.php | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index 19bffb2409f64..e6f1ebd8aef4e 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -188,16 +188,24 @@ private function seekToNextRelevantToken(\Iterator $tokenIterator) private function getArrayKeys(\Iterator $tokenIterator) { $keys = []; - - // Skip opening "[" - $tokenIterator->next(); + $isShortArray = '[' === $this->normalizeToken($tokenIterator->current()); + $closingBracket = $isShortArray ? ']' : ')'; + + if ($isShortArray) { + // Skip opening "[" + $tokenIterator->next(); + } else { + // Skip opening "array(" + $tokenIterator->next(); + $tokenIterator->next(); + } while ($tokenIterator->valid()) { $this->seekToNextRelevantToken($tokenIterator); $t = $tokenIterator->current(); // End of main array - if (']' === $t[0]) { + if ($closingBracket === $t[0]) { // Skip following "," $tokenIterator->next(); @@ -227,7 +235,7 @@ private function getArrayKeys(\Iterator $tokenIterator) break; } - if (-1 === $openBraces && ']' === $t[0]) { + if (-1 === $openBraces && $closingBracket === $t[0]) { break; } } From edb5aa9a4e8438c4ba0dfd882f41085ae4290743 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sat, 31 Oct 2020 00:42:12 +0100 Subject: [PATCH 08/23] Improving/fixing existing tests and adding variables tests (PHP extractor) --- .../Tests/Extractor/PhpExtractorTest.php | 172 +++++++++++++++--- .../extractor/translatable-fqn.html.php | 32 +++- .../extractor/translatable-short.html.php | 32 +++- .../fixtures/extractor/translatable.html.php | 32 +++- .../fixtures/extractor/translation.html.php | 36 +++- 5 files changed, 261 insertions(+), 43 deletions(-) diff --git a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php index 5b7ccaf7fde07..28b6f904af270 100644 --- a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php +++ b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php @@ -41,6 +41,7 @@ public function testExtraction($resource) // Assert $expectedCatalogue = [ 'messages' => [ + // translatable.html.php 'translatable single-quoted key' => 'prefixtranslatable single-quoted key', 'translatable double-quoted key' => 'prefixtranslatable double-quoted key', 'translatable heredoc key' => 'prefixtranslatable heredoc key', @@ -51,7 +52,20 @@ public function testExtraction($resource) 'translatable '.$expectedHeredoc => 'prefixtranslatable '.$expectedHeredoc, 'translatable '.$expectedNowdoc => 'prefixtranslatable '.$expectedNowdoc, 'translatable concatenated message with heredoc and nowdoc' => 'prefixtranslatable concatenated message with heredoc and nowdoc', - 'translatable default domain' => 'prefixtranslatable default domain', + 'translatable test-no-params-short-array' => 'prefixtranslatable test-no-params-short-array', + 'translatable test-no-params-long-array' => 'prefixtranslatable test-no-params-long-array', + 'translatable test-params-short-array' => 'prefixtranslatable test-params-short-array', + 'translatable test-params-long-array' => 'prefixtranslatable test-params-long-array', + 'translatable test-multiple-params-short-array' => 'prefixtranslatable test-multiple-params-short-array', + 'translatable test-multiple-params-long-array' => 'prefixtranslatable test-multiple-params-long-array', + 'translatable test-params-trailing-comma-short-array' => 'prefixtranslatable test-params-trailing-comma-short-array', + 'translatable test-params-trailing-comma-long-array' => 'prefixtranslatable test-params-trailing-comma-long-array', + 'translatable typecast-short-array' => 'prefixtranslatable typecast-short-array', + 'translatable typecast-long-array' => 'prefixtranslatable typecast-long-array', + 'translatable default-domain-short-array' => 'prefixtranslatable default-domain-short-array', + 'translatable default-domain-long-array' => 'prefixtranslatable default-domain-long-array', + + // translatable-fqn.html.php 'translatable-fqn single-quoted key' => 'prefixtranslatable-fqn single-quoted key', 'translatable-fqn double-quoted key' => 'prefixtranslatable-fqn double-quoted key', 'translatable-fqn heredoc key' => 'prefixtranslatable-fqn heredoc key', @@ -62,7 +76,20 @@ public function testExtraction($resource) 'translatable-fqn '.$expectedHeredoc => 'prefixtranslatable-fqn '.$expectedHeredoc, 'translatable-fqn '.$expectedNowdoc => 'prefixtranslatable-fqn '.$expectedNowdoc, 'translatable-fqn concatenated message with heredoc and nowdoc' => 'prefixtranslatable-fqn concatenated message with heredoc and nowdoc', - 'translatable-fqn default domain' => 'prefixtranslatable-fqn default domain', + 'translatable-fqn test-no-params-short-array' => 'prefixtranslatable-fqn test-no-params-short-array', + 'translatable-fqn test-no-params-long-array' => 'prefixtranslatable-fqn test-no-params-long-array', + 'translatable-fqn test-params-short-array' => 'prefixtranslatable-fqn test-params-short-array', + 'translatable-fqn test-params-long-array' => 'prefixtranslatable-fqn test-params-long-array', + 'translatable-fqn test-multiple-params-short-array' => 'prefixtranslatable-fqn test-multiple-params-short-array', + 'translatable-fqn test-multiple-params-long-array' => 'prefixtranslatable-fqn test-multiple-params-long-array', + 'translatable-fqn test-params-trailing-comma-short-array' => 'prefixtranslatable-fqn test-params-trailing-comma-short-array', + 'translatable-fqn test-params-trailing-comma-long-array' => 'prefixtranslatable-fqn test-params-trailing-comma-long-array', + 'translatable-fqn typecast-short-array' => 'prefixtranslatable-fqn typecast-short-array', + 'translatable-fqn typecast-long-array' => 'prefixtranslatable-fqn typecast-long-array', + 'translatable-fqn default-domain-short-array' => 'prefixtranslatable-fqn default-domain-short-array', + 'translatable-fqn default-domain-long-array' => 'prefixtranslatable-fqn default-domain-long-array', + + // translatable-short.html.php 'translatable-short single-quoted key' => 'prefixtranslatable-short single-quoted key', 'translatable-short double-quoted key' => 'prefixtranslatable-short double-quoted key', 'translatable-short heredoc key' => 'prefixtranslatable-short heredoc key', @@ -73,7 +100,20 @@ public function testExtraction($resource) 'translatable-short '.$expectedHeredoc => 'prefixtranslatable-short '.$expectedHeredoc, 'translatable-short '.$expectedNowdoc => 'prefixtranslatable-short '.$expectedNowdoc, 'translatable-short concatenated message with heredoc and nowdoc' => 'prefixtranslatable-short concatenated message with heredoc and nowdoc', - 'translatable-short default domain' => 'prefixtranslatable-short default domain', + 'translatable-short test-no-params-short-array' => 'prefixtranslatable-short test-no-params-short-array', + 'translatable-short test-no-params-long-array' => 'prefixtranslatable-short test-no-params-long-array', + 'translatable-short test-params-short-array' => 'prefixtranslatable-short test-params-short-array', + 'translatable-short test-params-long-array' => 'prefixtranslatable-short test-params-long-array', + 'translatable-short test-multiple-params-short-array' => 'prefixtranslatable-short test-multiple-params-short-array', + 'translatable-short test-multiple-params-long-array' => 'prefixtranslatable-short test-multiple-params-long-array', + 'translatable-short test-params-trailing-comma-short-array' => 'prefixtranslatable-short test-params-trailing-comma-short-array', + 'translatable-short test-params-trailing-comma-long-array' => 'prefixtranslatable-short test-params-trailing-comma-long-array', + 'translatable-short typecast-short-array' => 'prefixtranslatable-short typecast-short-array', + 'translatable-short typecast-long-array' => 'prefixtranslatable-short typecast-long-array', + 'translatable-short default-domain-short-array' => 'prefixtranslatable-short default-domain-short-array', + 'translatable-short default-domain-long-array' => 'prefixtranslatable-short default-domain-long-array', + + // translation.html.php 'single-quoted key' => 'prefixsingle-quoted key', 'double-quoted key' => 'prefixdouble-quoted key', 'heredoc key' => 'prefixheredoc key', @@ -84,35 +124,58 @@ public function testExtraction($resource) $expectedHeredoc => 'prefix'.$expectedHeredoc, $expectedNowdoc => 'prefix'.$expectedNowdoc, 'concatenated message with heredoc and nowdoc' => 'prefixconcatenated message with heredoc and nowdoc', - 'default domain' => 'prefixdefault domain', + 'test-no-params-short-array' => 'prefixtest-no-params-short-array', + 'test-no-params-long-array' => 'prefixtest-no-params-long-array', + 'test-params-short-array' => 'prefixtest-params-short-array', + 'test-params-long-array' => 'prefixtest-params-long-array', + 'test-multiple-params-short-array' => 'prefixtest-multiple-params-short-array', + 'test-multiple-params-long-array' => 'prefixtest-multiple-params-long-array', + 'test-params-trailing-comma-short-array' => 'prefixtest-params-trailing-comma-short-array', + 'test-params-trailing-comma-long-array' => 'prefixtest-params-trailing-comma-long-array', + 'typecast-short-array' => 'prefixtypecast-short-array', + 'typecast-long-array' => 'prefixtypecast-long-array', + 'default-domain-short-array' => 'prefixdefault-domain-short-array', + 'default-domain-long-array' => 'prefixdefault-domain-long-array', ], 'not_messages' => [ + // translatable.html.php 'translatable other-domain-test-no-params-short-array' => 'prefixtranslatable other-domain-test-no-params-short-array', 'translatable other-domain-test-no-params-long-array' => 'prefixtranslatable other-domain-test-no-params-long-array', 'translatable other-domain-test-params-short-array' => 'prefixtranslatable other-domain-test-params-short-array', 'translatable other-domain-test-params-long-array' => 'prefixtranslatable other-domain-test-params-long-array', - 'translatable typecast' => 'prefixtranslatable typecast', + 'translatable other-domain-typecast-short-array' => 'prefixtranslatable other-domain-typecast-short-array', + 'translatable other-domain-typecast-long-array' => 'prefixtranslatable other-domain-typecast-long-array', + + // translatable-fqn.html.php 'translatable-fqn other-domain-test-no-params-short-array' => 'prefixtranslatable-fqn other-domain-test-no-params-short-array', 'translatable-fqn other-domain-test-no-params-long-array' => 'prefixtranslatable-fqn other-domain-test-no-params-long-array', 'translatable-fqn other-domain-test-params-short-array' => 'prefixtranslatable-fqn other-domain-test-params-short-array', 'translatable-fqn other-domain-test-params-long-array' => 'prefixtranslatable-fqn other-domain-test-params-long-array', - 'translatable-fqn typecast' => 'prefixtranslatable-fqn typecast', + 'translatable-fqn other-domain-typecast-short-array' => 'prefixtranslatable-fqn other-domain-typecast-short-array', + 'translatable-fqn other-domain-typecast-long-array' => 'prefixtranslatable-fqn other-domain-typecast-long-array', + + // translatable-short.html.php 'translatable-short other-domain-test-no-params-short-array' => 'prefixtranslatable-short other-domain-test-no-params-short-array', 'translatable-short other-domain-test-no-params-long-array' => 'prefixtranslatable-short other-domain-test-no-params-long-array', 'translatable-short other-domain-test-params-short-array' => 'prefixtranslatable-short other-domain-test-params-short-array', 'translatable-short other-domain-test-params-long-array' => 'prefixtranslatable-short other-domain-test-params-long-array', - 'translatable-short typecast' => 'prefixtranslatable-short typecast', + 'translatable-short other-domain-typecast-short-array' => 'prefixtranslatable-short other-domain-typecast-short-array', + 'translatable-short other-domain-typecast-long-array' => 'prefixtranslatable-short other-domain-typecast-long-array', + + // translation.html.php 'other-domain-test-no-params-short-array' => 'prefixother-domain-test-no-params-short-array', 'other-domain-test-no-params-long-array' => 'prefixother-domain-test-no-params-long-array', 'other-domain-test-params-short-array' => 'prefixother-domain-test-params-short-array', 'other-domain-test-params-long-array' => 'prefixother-domain-test-params-long-array', - 'typecast' => 'prefixtypecast', + 'other-domain-typecast-short-array' => 'prefixother-domain-typecast-short-array', + 'other-domain-typecast-long-array' => 'prefixother-domain-typecast-long-array', ], ]; // Expected metadata (variables) $expectedVariables = [ 'messages' => [ + // translatable.html.php 'translatable single-quoted key' => null, 'translatable double-quoted key' => null, 'translatable heredoc key' => null, @@ -123,7 +186,20 @@ public function testExtraction($resource) 'translatable '.$expectedHeredoc => null, 'translatable '.$expectedNowdoc => null, 'translatable concatenated message with heredoc and nowdoc' => null, - 'translatable default domain' => null, + 'translatable test-no-params-short-array' => null, + 'translatable test-no-params-long-array' => null, + 'translatable test-params-short-array' => 'Available variables: foo', + 'translatable test-params-long-array' => 'Available variables: foo', + 'translatable test-multiple-params-short-array' => 'Available variables: foo, foz', + 'translatable test-multiple-params-long-array' => 'Available variables: foo, foz', + 'translatable test-params-trailing-comma-short-array' => 'Available variables: foo', + 'translatable test-params-trailing-comma-long-array' => 'Available variables: foo', + 'translatable typecast-short-array' => 'Available variables: a', + 'translatable typecast-long-array' => 'Available variables: a', + 'translatable default-domain-short-array' => null, + 'translatable default-domain-long-array' => null, + + // translatable-fqn.html.php 'translatable-fqn single-quoted key' => null, 'translatable-fqn double-quoted key' => null, 'translatable-fqn heredoc key' => null, @@ -134,7 +210,20 @@ public function testExtraction($resource) 'translatable-fqn '.$expectedHeredoc => null, 'translatable-fqn '.$expectedNowdoc => null, 'translatable-fqn concatenated message with heredoc and nowdoc' => null, - 'translatable-fqn default domain' => null, + 'translatable-fqn test-no-params-short-array' => null, + 'translatable-fqn test-no-params-long-array' => null, + 'translatable-fqn test-params-short-array' => 'Available variables: foo', + 'translatable-fqn test-params-long-array' => 'Available variables: foo', + 'translatable-fqn test-multiple-params-short-array' => 'Available variables: foo, foz', + 'translatable-fqn test-multiple-params-long-array' => 'Available variables: foo, foz', + 'translatable-fqn test-params-trailing-comma-short-array' => 'Available variables: foo', + 'translatable-fqn test-params-trailing-comma-long-array' => 'Available variables: foo', + 'translatable-fqn typecast-short-array' => 'Available variables: a', + 'translatable-fqn typecast-long-array' => 'Available variables: a', + 'translatable-fqn default-domain-short-array' => null, + 'translatable-fqn default-domain-long-array' => null, + + // translatable-short.html.php 'translatable-short single-quoted key' => null, 'translatable-short double-quoted key' => null, 'translatable-short heredoc key' => null, @@ -145,7 +234,20 @@ public function testExtraction($resource) 'translatable-short '.$expectedHeredoc => null, 'translatable-short '.$expectedNowdoc => null, 'translatable-short concatenated message with heredoc and nowdoc' => null, - 'translatable-short default domain' => null, + 'translatable-short test-no-params-short-array' => null, + 'translatable-short test-no-params-long-array' => null, + 'translatable-short test-params-short-array' => 'Available variables: foo', + 'translatable-short test-params-long-array' => 'Available variables: foo', + 'translatable-short test-multiple-params-short-array' => 'Available variables: foo, foz', + 'translatable-short test-multiple-params-long-array' => 'Available variables: foo, foz', + 'translatable-short test-params-trailing-comma-short-array' => 'Available variables: foo', + 'translatable-short test-params-trailing-comma-long-array' => 'Available variables: foo', + 'translatable-short typecast-short-array' => 'Available variables: a', + 'translatable-short typecast-long-array' => 'Available variables: a', + 'translatable-short default-domain-short-array' => null, + 'translatable-short default-domain-long-array' => null, + + // translation.html.php 'single-quoted key' => null, 'double-quoted key' => null, 'heredoc key' => null, @@ -156,29 +258,51 @@ public function testExtraction($resource) $expectedHeredoc => null, $expectedNowdoc => null, 'concatenated message with heredoc and nowdoc' => null, - 'default domain' => null, + 'test-no-params-short-array' => null, + 'test-no-params-long-array' => null, + 'test-params-short-array' => 'Available variables: foo', + 'test-params-long-array' => 'Available variables: foo', + 'test-multiple-params-short-array' => 'Available variables: foo, foz', + 'test-multiple-params-long-array' => 'Available variables: foo, foz', + 'test-params-trailing-comma-short-array' => 'Available variables: foo', + 'test-params-trailing-comma-long-array' => 'Available variables: foo', + 'typecast-short-array' => 'Available variables: a', + 'typecast-long-array' => 'Available variables: a', + 'default-domain-short-array' => null, + 'default-domain-long-array' => null, ], 'not_messages' => [ + // translatable.html.php 'translatable other-domain-test-no-params-short-array' => null, 'translatable other-domain-test-no-params-long-array' => null, 'translatable other-domain-test-params-short-array' => 'Available variables: foo', 'translatable other-domain-test-params-long-array' => 'Available variables: foo', - 'translatable typecast' => 'Available variables: a', + 'translatable other-domain-typecast-short-array' => 'Available variables: a', + 'translatable other-domain-typecast-long-array' => 'Available variables: a', + + // translatable-fqn.html.php 'translatable-fqn other-domain-test-no-params-short-array' => null, 'translatable-fqn other-domain-test-no-params-long-array' => null, 'translatable-fqn other-domain-test-params-short-array' => 'Available variables: foo', 'translatable-fqn other-domain-test-params-long-array' => 'Available variables: foo', - 'translatable-fqn typecast' => 'Available variables: a', + 'translatable-fqn other-domain-typecast-short-array' => 'Available variables: a', + 'translatable-fqn other-domain-typecast-long-array' => 'Available variables: a', + + // translatable-short.html.php 'translatable-short other-domain-test-no-params-short-array' => null, 'translatable-short other-domain-test-no-params-long-array' => null, 'translatable-short other-domain-test-params-short-array' => 'Available variables: foo', 'translatable-short other-domain-test-params-long-array' => 'Available variables: foo', - 'translatable-short typecast' => 'Available variables: a', + 'translatable-short other-domain-typecast-short-array' => 'Available variables: a', + 'translatable-short other-domain-typecast-long-array' => 'Available variables: a', + + // translation.html.php 'other-domain-test-no-params-short-array' => null, 'other-domain-test-no-params-long-array' => null, 'other-domain-test-params-short-array' => 'Available variables: foo', 'other-domain-test-params-long-array' => 'Available variables: foo', - 'typecast' => 'Available variables: a', + 'other-domain-typecast-short-array' => 'Available variables: a', + 'other-domain-typecast-long-array' => 'Available variables: a', ], ]; $actualCatalogue = $catalogue->all(); @@ -187,30 +311,28 @@ public function testExtraction($resource) $filename = str_replace(\DIRECTORY_SEPARATOR, '/', __DIR__).'/../fixtures/extractor/translatable.html.php'; $this->assertEquals(['sources' => [$filename.':2']], $catalogue->getMetadata('translatable single-quoted key')); - $this->assertEquals(['sources' => [$filename.':37']], $catalogue->getMetadata('translatable other-domain-test-no-params-short-array', 'not_messages')); + $this->assertEquals(['sources' => [$filename.':57']], $catalogue->getMetadata('translatable other-domain-test-no-params-short-array', 'not_messages')); $filename = str_replace(\DIRECTORY_SEPARATOR, '/', __DIR__).'/../fixtures/extractor/translatable-fqn.html.php'; $this->assertEquals(['sources' => [$filename.':2']], $catalogue->getMetadata('translatable-fqn single-quoted key')); - $this->assertEquals(['sources' => [$filename.':37']], $catalogue->getMetadata('translatable-fqn other-domain-test-no-params-short-array', 'not_messages')); + $this->assertEquals(['sources' => [$filename.':57']], $catalogue->getMetadata('translatable-fqn other-domain-test-no-params-short-array', 'not_messages')); $filename = str_replace(\DIRECTORY_SEPARATOR, '/', __DIR__).'/../fixtures/extractor/translatable-short.html.php'; $this->assertEquals(['sources' => [$filename.':2']], $catalogue->getMetadata('translatable-short single-quoted key')); - $this->assertEquals(['sources' => [$filename.':37']], $catalogue->getMetadata('translatable-short other-domain-test-no-params-short-array', 'not_messages')); + $this->assertEquals(['sources' => [$filename.':57']], $catalogue->getMetadata('translatable-short other-domain-test-no-params-short-array', 'not_messages')); $filename = str_replace(\DIRECTORY_SEPARATOR, '/', __DIR__).'/../fixtures/extractor/translation.html.php'; $this->assertEquals(['sources' => [$filename.':2']], $catalogue->getMetadata('single-quoted key')); - $this->assertEquals(['sources' => [$filename.':37']], $catalogue->getMetadata('other-domain-test-no-params-short-array', 'not_messages')); + $this->assertEquals(['sources' => [$filename.':57']], $catalogue->getMetadata('other-domain-test-no-params-short-array', 'not_messages')); // Check variables metadata + $extractedVariables = []; foreach (array_keys($actualCatalogue) as $domain) { foreach (array_keys($actualCatalogue[$domain]) as $id) { - $this->assertArrayHasKey($id, $expectedVariables[$domain], 'Metadata for domain "'.$domain.'" and id "'.$id.'" was not expected!'); - - $extractedVariables = $this->getVariablesNoteContentFromMetadata($catalogue->getMetadata($id, $domain)); - - $this->assertEquals($expectedVariables[$domain][$id], $extractedVariables); + $extractedVariables[$domain][$id] = $this->getVariablesNoteContentFromMetadata($catalogue->getMetadata($id, $domain)); } } + $this->assertEquals($expectedVariables, $extractedVariables); } /** diff --git a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable-fqn.html.php b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable-fqn.html.php index 87a64c42f1eec..d5b6b62186ef4 100644 --- a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable-fqn.html.php +++ b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable-fqn.html.php @@ -34,14 +34,38 @@ EOF ); ?> + + + + + 'bar']); ?> + + 'bar')); ?> + + 'bar', 'foz' => 'baz']); ?> + + 'bar', 'foz' => 'baz')); ?> + + 'bar',]); ?> + + 'bar',)); ?> + + (int) '123']); ?> + + (int) '123')); ?> + - + 'bar'], 'not_messages'); ?> - 'bar'], 'not_messages'); ?> + 'bar'), 'not_messages'); ?> + + (int) '123'], 'not_messages'); ?> + + (int) '123'), 'not_messages'); ?> - (int) '123'], 'not_messages'); ?> + - + diff --git a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable-short.html.php b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable-short.html.php index d8842b97f1ada..ddfc10f223446 100644 --- a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable-short.html.php +++ b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable-short.html.php @@ -34,14 +34,38 @@ EOF ); ?> + + + + + 'bar']); ?> + + 'bar')); ?> + + 'bar', 'foz' => 'baz']); ?> + + 'bar', 'foz' => 'baz')); ?> + + 'bar',]); ?> + + 'bar',)); ?> + + (int) '123']); ?> + + (int) '123')); ?> + - + 'bar'], 'not_messages'); ?> - 'bar'], 'not_messages'); ?> + 'bar'), 'not_messages'); ?> + + (int) '123'], 'not_messages'); ?> + + (int) '123'), 'not_messages'); ?> - (int) '123'], 'not_messages'); ?> + - + diff --git a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable.html.php b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable.html.php index 828707e26ed02..f7dd6025f8024 100644 --- a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable.html.php +++ b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translatable.html.php @@ -34,14 +34,38 @@ EOF ); ?> + + + + + 'bar']); ?> + + 'bar')); ?> + + 'bar', 'foz' => 'baz']); ?> + + 'bar', 'foz' => 'baz')); ?> + + 'bar',]); ?> + + 'bar',)); ?> + + (int) '123']); ?> + + (int) '123')); ?> + - + 'bar'], 'not_messages'); ?> - 'bar'], 'not_messages'); ?> + 'bar'), 'not_messages'); ?> + + (int) '123'], 'not_messages'); ?> + + (int) '123'), 'not_messages'); ?> - (int) '123'], 'not_messages'); ?> + - + diff --git a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php index a6adda0565628..87a050a4a297f 100644 --- a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php +++ b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php @@ -34,14 +34,38 @@ EOF ); ?> -trans('other-domain-test-no-params-short-array', [], 'not_messages'); ?> +trans('test-no-params-short-array', []); ?> -trans('other-domain-test-no-params-long-array', [], 'not_messages'); ?> +trans('test-no-params-long-array', array()); ?> -trans('other-domain-test-params-short-array', ['foo' => 'bar'], 'not_messages'); ?> +trans('test-params-short-array', ['foo' => 'bar']); ?> -trans('other-domain-test-params-long-array', ['foo' => 'bar'], 'not_messages'); ?> +trans('test-params-long-array', array('foo' => 'bar')); ?> -trans('typecast', ['a' => (int) '123'], 'not_messages'); ?> +trans('test-multiple-params-short-array', ['foo' => 'bar', 'foz' => 'baz']); ?> -trans('default domain', [], null); ?> +trans('test-multiple-params-long-array', array('foo' => 'bar', 'foz' => 'baz')); ?> + +trans('test-params-trailing-comma-short-array', ['foo' => 'bar',]); ?> + +trans('test-params-trailing-comma-long-array', array('foo' => 'bar',)); ?> + +trans('typecast-short-array', ['a' => (int) '123']); ?> + +trans('typecast-long-array', array('a' => (int) '123')); ?> + +trans('other-domain-test-no-params-short-array', [], 'not_messages'); ?> + +trans('other-domain-test-no-params-long-array', array(), 'not_messages'); ?> + +trans('other-domain-test-params-short-array', ['foo' => 'bar'], 'not_messages'); ?> + +trans('other-domain-test-params-long-array', array('foo' => 'bar'), 'not_messages'); ?> + +trans('other-domain-typecast-short-array', ['a' => (int) '123'], 'not_messages'); ?> + +trans('other-domain-typecast-long-array', array('a' => (int) '123'), 'not_messages'); ?> + +trans('default-domain-short-array', [], null); ?> + +trans('default-domain-long-array', array(), null); ?> From 31f96a0990f41a957ec2366df5a32ff968e64663 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sat, 31 Oct 2020 03:01:43 +0100 Subject: [PATCH 09/23] Adding variables tests (Twig extractor) --- .../NodeVisitor/TranslationNodeVisitor.php | 4 +- .../Tests/Translation/TwigExtractorTest.php | 97 ++++++++++++++----- 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index c99fd161b4bce..6a429ea29c44c 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -40,8 +40,8 @@ final class TranslationNodeVisitor extends AbstractNodeVisitor * 0 => 'message', * 1 => 'domain', * 2 => [ - * 'variable1', - * 'variable2', + * 0 => 'variable1', + * 1 => 'variable2', * ... * ] * ], diff --git a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php index 6a7336d7b1995..0b3cd8ceda90b 100644 --- a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php +++ b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php @@ -22,6 +22,9 @@ class TwigExtractorTest extends TestCase { + const VARIABLES_NOTE_CATEGORY = 'symfony-extractor-variables'; + const VARIABLES_NOTE_PREFIX = 'Available variables: '; + /** * @dataProvider getExtractData */ @@ -48,42 +51,92 @@ public function testExtract($template, $messages) $this->assertSame($catalogue->all(), $messages); } - foreach ($messages as $key => $domain) { - $this->assertTrue($catalogue->has($key, $domain)); - $this->assertEquals('prefix'.$key, $catalogue->get($key, $domain)); + foreach ($messages as $key => $data) { + $this->assertTrue($catalogue->has($key, $data[0])); + $this->assertEquals('prefix'.$key, $catalogue->get($key, $data[0])); + + // Check variables + if ($notes = $catalogue->getMetadata($key, $data[0])['notes'] ?? null) { + foreach ($notes as $note) { + if (isset($note['category']) && self::VARIABLES_NOTE_CATEGORY === $note['category']) { + $this->assertEquals(self::VARIABLES_NOTE_PREFIX.$data[1], $note['content']); + + break; + } + } + } } } public function getExtractData() { + /* + * [ + * 0 => 'TWIG_TEMPLATE', + * 1 => [ + * 'message_key' => [ + * 0 => 'domain', + * 1 => 'var1, var2'|null, // Complete message is 'Available variables: var1, var2' + * ... + * ] + * ], + * ... + * ] + */ return [ - ['{{ "new key" | trans() }}', ['new key' => 'messages']], - ['{{ "new key" | trans() | upper }}', ['new key' => 'messages']], - ['{{ "new key" | trans({}, "domain") }}', ['new key' => 'domain']], - ['{% trans %}new key{% endtrans %}', ['new key' => 'messages']], - ['{% trans %} new key {% endtrans %}', ['new key' => 'messages']], - ['{% trans from "domain" %}new key{% endtrans %}', ['new key' => 'domain']], - ['{% set foo = "new key" | trans %}', ['new key' => 'messages']], - ['{{ 1 ? "new key" | trans : "another key" | trans }}', ['new key' => 'messages', 'another key' => 'messages']], - ['{{ t("new key") | trans() }}', ['new key' => 'messages']], - ['{{ t("new key", {}, "domain") | trans() }}', ['new key' => 'domain']], - ['{{ 1 ? t("new key") | trans : t("another key") | trans }}', ['new key' => 'messages', 'another key' => 'messages']], + ['{{ "new key" | trans() }}', ['new key' => ['messages', null]]], + ['{{ "new key" | trans() | upper }}', ['new key' => ['messages', null]]], + ['{{ "new key" | trans({}, "domain") }}', ['new key' => ['domain', null]]], + ['{% trans %}new key{% endtrans %}', ['new key' => ['messages', null]]], + ['{% trans %} new key {% endtrans %}', ['new key' => ['messages', null]]], + ['{% trans from "domain" %}new key{% endtrans %}', ['new key' => ['domain', null]]], + ['{% set foo = "new key" | trans %}', ['new key' => ['messages', null]]], + ['{{ 1 ? "new key" | trans : "another key" | trans }}', ['new key' => ['messages', null], 'another key' => ['messages', null]]], + ['{{ t("new key") | trans() }}', ['new key' => ['messages', null]]], + ['{{ t("new key", {}, "domain") | trans() }}', ['new key' => ['domain', null]]], + ['{{ 1 ? t("new key") | trans : t("another key") | trans }}', ['new key' => ['messages', null], 'another key' => ['messages', null]]], // make sure 'trans_default_domain' tag is supported - ['{% trans_default_domain "domain" %}{{ "new key"|trans }}', ['new key' => 'domain']], - ['{% trans_default_domain "domain" %}{% trans %}new key{% endtrans %}', ['new key' => 'domain']], + ['{% trans_default_domain "domain" %}{{ "new key"|trans }}', ['new key' => ['domain', null]]], + ['{% trans_default_domain "domain" %}{% trans %}new key{% endtrans %}', ['new key' => ['domain', null]]], // make sure this works with twig's named arguments - ['{{ "new key" | trans(domain="domain") }}', ['new key' => 'domain']], + ['{{ "new key" | trans(domain="domain") }}', ['new key' => ['domain', null]]], + + // make sure this works with variables + // trans tag + ['{% trans with {\'var1\': \'val1\', \'var2\': \'val2\'} %}trans_tag_with_variables{% endtrans %}', ['trans_tag_with_variables' => ['messages', 'var1, var2']]], + ['{% trans with {\'var1\': \'val1\', \'var2\': \'val2\'} from \'domain\' %}trans_tag_with_variables_and_domain{% endtrans %}', ['trans_tag_with_variables_and_domain' => ['domain', 'var1, var2']]], + ['{% trans with {\'var1\': \'val1\', \'var2\': \'val2\'} from \'domain\' into \'en\'%}trans_tag_with_variables_and_domain_and_locale{% endtrans %}', ['trans_tag_with_variables_and_domain_and_locale' => ['domain', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{% trans with {\'var1\': \'val1\', \'var2\': \'val2\'} %}trans_tag_with_variables{% endtrans %}', ['trans_tag_with_variables' => ['another-domain', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{% trans with {\'var1\': \'val1\', \'var2\': \'val2\'} from \'domain\' %}trans_tag_with_variables_and_domain{% endtrans %}', ['trans_tag_with_variables_and_domain' => ['domain', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{% trans with {\'var1\': \'val1\', \'var2\': \'val2\'} from \'domain\' into \'en\'%}trans_tag_with_variables_and_domain_and_locale{% endtrans %}', ['trans_tag_with_variables_and_domain_and_locale' => ['domain', 'var1, var2']]], + + // |trans() filter + ['{{ \'trans_filter_with_variable_as_param\'|trans({\'var1\': \'val1\', \'var2\': \'val2\'}) }}', ['trans_filter_with_variable_as_param' => ['messages', 'var1, var2']]], + ['{{ \'trans_filter_with_variable_as_param\'|trans({\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\') }}', ['trans_filter_with_variable_as_param' => ['domain', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{{ \'trans_filter_with_variable_as_param\'|trans({\'var1\': \'val1\', \'var2\': \'val2\'}) }}', ['trans_filter_with_variable_as_param' => ['another-domain', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{{ \'trans_filter_with_variable_as_param\'|trans({\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\') }}', ['trans_filter_with_variable_as_param' => ['domain', 'var1, var2']]], + + // t() function + // Be careful: the t() function creates a TranslatableMessage object which overrides the domain set with 'trans_default_domain' (no domain specified = 'messages'). + ['{{ t(\'t_function_with_variables_with_trans_filter\', {\'var1\': \'val1\', \'var2\': \'val2\'})|trans }}', ['t_function_with_variables_with_trans_filter' => ['messages', 'var1, var2']]], + ['{{ t(\'t_function_with_variables_and_domain_with_trans_filter\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\')|trans }}', ['t_function_with_variables_and_domain_with_trans_filter' => ['domain', 'var1, var2']]], + ['{{ t(\'t_function_with_variables_with_trans_filter_and_locale\', {\'var1\': \'val1\', \'var2\': \'val2\'})|trans(\'en\') }}', ['t_function_with_variables_with_trans_filter_and_locale' => ['messages', 'var1, var2']]], + ['{{ t(\'t_function_with_variables_and_domain_with_trans_filter_and_locale\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\')|trans(\'en\') }}', ['t_function_with_variables_and_domain_with_trans_filter_and_locale' => ['domain', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{{ t(\'t_function_with_variables_with_trans_filter\', {\'var1\': \'val1\', \'var2\': \'val2\'})|trans }}', ['t_function_with_variables_with_trans_filter' => ['messages', 'var1, var2']]], // Be careful! (read note above) + ['{% trans_default_domain \'another-domain\' %}{{ t(\'t_function_with_variables_and_domain_with_trans_filter\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\')|trans }}', ['t_function_with_variables_and_domain_with_trans_filter' => ['domain', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{{ t(\'t_function_with_variables_with_trans_filter_and_locale\', {\'var1\': \'val1\', \'var2\': \'val2\'})|trans(\'en\') }}', ['t_function_with_variables_with_trans_filter_and_locale' => ['messages', 'var1, var2']]], // Be careful! (read note above) + ['{% trans_default_domain \'another-domain\' %}{{ t(\'t_function_with_variables_and_domain_with_trans_filter_and_locale\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\')|trans(\'en\') }}', ['t_function_with_variables_and_domain_with_trans_filter_and_locale' => ['domain', 'var1, var2']]], // concat translations - ['{{ ("new" ~ " key") | trans() }}', ['new key' => 'messages']], - ['{{ ("another " ~ "new " ~ "key") | trans() }}', ['another new key' => 'messages']], - ['{{ ("new" ~ " key") | trans(domain="domain") }}', ['new key' => 'domain']], - ['{{ ("another " ~ "new " ~ "key") | trans(domain="domain") }}', ['another new key' => 'domain']], + ['{{ ("new" ~ " key") | trans() }}', ['new key' => ['messages', null]]], + ['{{ ("another " ~ "new " ~ "key") | trans() }}', ['another new key' => ['messages', null]]], + ['{{ ("new" ~ " key") | trans(domain="domain") }}', ['new key' => ['domain', null]]], + ['{{ ("another " ~ "new " ~ "key") | trans(domain="domain") }}', ['another new key' => ['domain', null]]], // if it has a variable or other expression, we can not extract it ['{% set foo = "new" %} {{ ("new " ~ foo ~ "key") | trans() }}', []], - ['{{ ("foo " ~ "new"|trans ~ "key") | trans() }}', ['new' => 'messages']], + ['{{ ("foo " ~ "new"|trans ~ "key") | trans() }}', ['new' => ['messages', null]]], ]; } From 5c2e78253668211bab034afcc44424e9df7ec905 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sat, 31 Oct 2020 11:17:03 +0100 Subject: [PATCH 10/23] Clearer naming --- .../Component/Translation/Extractor/PhpExtractor.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index e6f1ebd8aef4e..134b772f723db 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -189,7 +189,7 @@ private function getArrayKeys(\Iterator $tokenIterator) { $keys = []; $isShortArray = '[' === $this->normalizeToken($tokenIterator->current()); - $closingBracket = $isShortArray ? ']' : ')'; + $mainClosingSymbol = $isShortArray ? ']' : ')'; if ($isShortArray) { // Skip opening "[" @@ -205,7 +205,7 @@ private function getArrayKeys(\Iterator $tokenIterator) $t = $tokenIterator->current(); // End of main array - if ($closingBracket === $t[0]) { + if ($mainClosingSymbol === $t[0]) { // Skip following "," $tokenIterator->next(); @@ -235,7 +235,7 @@ private function getArrayKeys(\Iterator $tokenIterator) break; } - if (-1 === $openBraces && $closingBracket === $t[0]) { + if (-1 === $openBraces && $mainClosingSymbol === $t[0]) { break; } } From c3e2e883ec5bc84d8676ba6e3d8c07b8628b8567 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sat, 31 Oct 2020 12:18:23 +0100 Subject: [PATCH 11/23] Avoid overwriting metadata completely when adding variables note (Twig extractor) --- .../Bridge/Twig/Translation/TwigExtractor.php | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php index 6c8c6a01d6f1a..a29d64d5a9842 100644 --- a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php +++ b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php @@ -77,18 +77,21 @@ protected function extractTemplate(string $template, MessageCatalogue $catalogue $this->twig->parse($this->twig->tokenize(new Source($template, ''))); - foreach ($visitor->getMessages() as $message) { - $id = trim($message[0]); - - $catalogue->set($id, $this->prefix.trim($message[0]), $message[1] ?: $this->defaultDomain); - - if (!empty($message[2])) { - $catalogue->setMetadata($id, ['notes' => [ - [ - 'category' => 'symfony-extractor-variables', - 'content' => 'Available variables: '.implode(', ', $message[2]), - ], - ]], $message[1] ?: $this->defaultDomain); + foreach ($visitor->getMessages() as $extractedMessage) { + $message = trim($extractedMessage[0]); + $domain = $extractedMessage[1] ?: $this->defaultDomain; + + $catalogue->set($message, $this->prefix.trim($extractedMessage[0]), $domain); + + if (!empty($extractedMessage[2])) { + $metadata = $catalogue->getMetadata($message, $domain); + + $metadata['notes'][] = [ + 'category' => 'symfony-extractor-variables', + 'content' => 'Available variables: '.implode(', ', $extractedMessage[2]), + ]; + + $catalogue->setMetadata($message, $metadata, $domain); } } From c50ce675b390d4cb23179b76d864c60f7aac4cf7 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sat, 31 Oct 2020 18:23:38 +0100 Subject: [PATCH 12/23] Supporting "t()" functions (Twig) without "|trans()" filter (e.g. as variable value) --- .../NodeVisitor/TranslationNodeVisitor.php | 30 +++++++++++++++++++ .../Tests/Translation/TwigExtractorTest.php | 11 ++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index 6a429ea29c44c..72fc5c5d3656a 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -30,6 +30,13 @@ final class TranslationNodeVisitor extends AbstractNodeVisitor public const UNDEFINED_DOMAIN = '_undefined'; private $enabled = false; + /** + * This class cannot read nodes backwards. + * We need a way to track when we encounter a "|trans()" filter containing + * a "t()" function in order to avoid processing that same function again + * as if it was used alone. + */ + private $skipTFunctionAfterFilter = false; /** * This array stores found messages. * @@ -96,6 +103,29 @@ protected function doEnterNode(Node $node, Environment $env): Node // extract t() nodes with a trans filter applied $functionNodeArguments = $node->getNode('node')->getNode('arguments'); + if ($functionNodeArguments->getIterator()->current() instanceof ConstantExpression) { + $this->messages[] = [ + $this->getReadMessageFromArguments($functionNodeArguments, 0), + $this->getReadDomainFromArguments($functionNodeArguments, 2), + $this->getReadVariablesFromArguments($functionNodeArguments, 1), + ]; + + // Avoid processing this "t()" function twice + $this->skipTFunctionAfterFilter = true; + } + } elseif ( + $node instanceof FunctionExpression && + 't' === $node->getAttribute('name') + ) { + // extract t() nodes without a trans filter applied + if ($this->skipTFunctionAfterFilter) { + $this->skipTFunctionAfterFilter = false; + + return $node; + } + + $functionNodeArguments = $node->getNode('arguments'); + if ($functionNodeArguments->getIterator()->current() instanceof ConstantExpression) { $this->messages[] = [ $this->getReadMessageFromArguments($functionNodeArguments, 0), diff --git a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php index 0b3cd8ceda90b..2b0d4759760ce 100644 --- a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php +++ b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php @@ -118,7 +118,7 @@ public function getExtractData() ['{% trans_default_domain \'another-domain\' %}{{ \'trans_filter_with_variable_as_param\'|trans({\'var1\': \'val1\', \'var2\': \'val2\'}) }}', ['trans_filter_with_variable_as_param' => ['another-domain', 'var1, var2']]], ['{% trans_default_domain \'another-domain\' %}{{ \'trans_filter_with_variable_as_param\'|trans({\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\') }}', ['trans_filter_with_variable_as_param' => ['domain', 'var1, var2']]], - // t() function + // t() function with |trans() filter // Be careful: the t() function creates a TranslatableMessage object which overrides the domain set with 'trans_default_domain' (no domain specified = 'messages'). ['{{ t(\'t_function_with_variables_with_trans_filter\', {\'var1\': \'val1\', \'var2\': \'val2\'})|trans }}', ['t_function_with_variables_with_trans_filter' => ['messages', 'var1, var2']]], ['{{ t(\'t_function_with_variables_and_domain_with_trans_filter\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\')|trans }}', ['t_function_with_variables_and_domain_with_trans_filter' => ['domain', 'var1, var2']]], @@ -129,6 +129,15 @@ public function getExtractData() ['{% trans_default_domain \'another-domain\' %}{{ t(\'t_function_with_variables_with_trans_filter_and_locale\', {\'var1\': \'val1\', \'var2\': \'val2\'})|trans(\'en\') }}', ['t_function_with_variables_with_trans_filter_and_locale' => ['messages', 'var1, var2']]], // Be careful! (read note above) ['{% trans_default_domain \'another-domain\' %}{{ t(\'t_function_with_variables_and_domain_with_trans_filter_and_locale\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\')|trans(\'en\') }}', ['t_function_with_variables_and_domain_with_trans_filter_and_locale' => ['domain', 'var1, var2']]], + // t() function alone (e.g. as a variable value) + // Be careful: the t() function creates a TranslatableMessage object which overrides the domain set with 'trans_default_domain' (no domain specified = 'messages'). + ['{% set t_function_in_a_variable = t(\'t_function_in_a_variable\') %}{{ t_function_in_a_variable|trans }}', ['t_function_in_a_variable' => ['messages', null]]], + ['{% set t_function_with_variables_in_a_variable = t(\'t_function_with_variables_in_a_variable\', {\'var1\': \'val1\', \'var2\': \'val2\'}) %}{{ t_function_with_variables_in_a_variable|trans }}', ['t_function_with_variables_in_a_variable' => ['messages', 'var1, var2']]], + ['{% set t_function_with_variables_and_domain_in_a_variable = t(\'t_function_with_variables_and_domain_in_a_variable\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\') %}{{ t_function_with_variables_and_domain_in_a_variable|trans }}', ['t_function_with_variables_and_domain_in_a_variable' => ['domain', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{% set t_function_in_a_variable = t(\'t_function_in_a_variable\') %}{{ t_function_in_a_variable|trans }}', ['t_function_in_a_variable' => ['messages', null]]], + ['{% trans_default_domain \'another-domain\' %}{% set t_function_with_variables_in_a_variable = t(\'t_function_with_variables_in_a_variable\', {\'var1\': \'val1\', \'var2\': \'val2\'}) %}{{ t_function_with_variables_in_a_variable|trans }}', ['t_function_with_variables_in_a_variable' => ['messages', 'var1, var2']]], + ['{% trans_default_domain \'another-domain\' %}{% set t_function_with_variables_and_domain_in_a_variable = t(\'t_function_with_variables_and_domain_in_a_variable\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\') %}{{ t_function_with_variables_and_domain_in_a_variable|trans }}', ['t_function_with_variables_and_domain_in_a_variable' => ['domain', 'var1, var2']]], + // concat translations ['{{ ("new" ~ " key") | trans() }}', ['new key' => ['messages', null]]], ['{{ ("another " ~ "new " ~ "key") | trans() }}', ['another new key' => ['messages', null]]], From 6e4013c4246a016431761862fcc57d4cab0f3610 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sat, 31 Oct 2020 23:33:11 +0100 Subject: [PATCH 13/23] Fixing duplicate notes. Extractors now keep the higher variables count between occurrences of the same key --- .../Tests/Translation/TwigExtractorTest.php | 3 +++ .../Bridge/Twig/Translation/TwigExtractor.php | 19 +++++++++++++++++-- .../Translation/Extractor/PhpExtractor.php | 18 +++++++++++++++++- .../Tests/Extractor/PhpExtractorTest.php | 2 ++ .../fixtures/extractor/translation.html.php | 5 +++++ 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php index 2b0d4759760ce..873465b87ae2e 100644 --- a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php +++ b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php @@ -138,6 +138,9 @@ public function getExtractData() ['{% trans_default_domain \'another-domain\' %}{% set t_function_with_variables_in_a_variable = t(\'t_function_with_variables_in_a_variable\', {\'var1\': \'val1\', \'var2\': \'val2\'}) %}{{ t_function_with_variables_in_a_variable|trans }}', ['t_function_with_variables_in_a_variable' => ['messages', 'var1, var2']]], ['{% trans_default_domain \'another-domain\' %}{% set t_function_with_variables_and_domain_in_a_variable = t(\'t_function_with_variables_and_domain_in_a_variable\', {\'var1\': \'val1\', \'var2\': \'val2\'}, \'domain\') %}{{ t_function_with_variables_and_domain_in_a_variable|trans }}', ['t_function_with_variables_and_domain_in_a_variable' => ['domain', 'var1, var2']]], + // Check behavior when the same key is used multiple times (no duplicate notes, keep higher variables count) + ['{% trans with {\'var1\': \'val1\', \'var2\': \'val2\'} %}message_used_multiple_times{% endtrans %}{% trans with {\'var1\': \'val1\', \'var2\': \'val2\', \'var3\': \'val3\'} %}message_used_multiple_times{% endtrans %}{% trans with {\'var1\': \'val1\'} %}message_used_multiple_times{% endtrans %}', ['message_used_multiple_times' => ['messages', 'var1, var2, var3']]], + // concat translations ['{{ ("new" ~ " key") | trans() }}', ['new key' => ['messages', null]]], ['{{ ("another " ~ "new " ~ "key") | trans() }}', ['another new key' => ['messages', null]]], diff --git a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php index a29d64d5a9842..ab2cece5de392 100644 --- a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php +++ b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php @@ -85,12 +85,27 @@ protected function extractTemplate(string $template, MessageCatalogue $catalogue if (!empty($extractedMessage[2])) { $metadata = $catalogue->getMetadata($message, $domain); - - $metadata['notes'][] = [ + $variablesNote = [ 'category' => 'symfony-extractor-variables', 'content' => 'Available variables: '.implode(', ', $extractedMessage[2]), ]; + // Update old variables note (if any) + if (isset($metadata['notes'])) { + foreach ($metadata['notes'] as $index => $note) { + if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { + // Keep the higher variables count + if (count($extractedMessage[2]) > substr_count($note['content'], ',')) { + $metadata['notes'][$index] = $variablesNote; + } + + break; + } + } + } else { + $metadata['notes'][] = $variablesNote; + } + $catalogue->setMetadata($message, $metadata, $domain); } } diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index 134b772f723db..942251194b61a 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -351,10 +351,26 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string $metadata['sources'][] = $normalizedFilename.':'.$tokens[$key][2]; if (!empty($variables)) { - $metadata['notes'][] = [ + $variablesNote = [ 'category' => 'symfony-extractor-variables', 'content' => 'Available variables: '.implode(', ', $variables), ]; + + // Update old variables note (if any) + if (isset($metadata['notes'])) { + foreach ($metadata['notes'] as $index => $note) { + if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { + // Keep the higher variables count + if (count($variables) > substr_count($note['content'], ',')) { + $metadata['notes'][$index] = $variablesNote; + } + + break; + } + } + } else { + $metadata['notes'][] = $variablesNote; + } } $catalog->setMetadata($message, $metadata, $domain); diff --git a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php index 28b6f904af270..8a418775886b0 100644 --- a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php +++ b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php @@ -136,6 +136,7 @@ public function testExtraction($resource) 'typecast-long-array' => 'prefixtypecast-long-array', 'default-domain-short-array' => 'prefixdefault-domain-short-array', 'default-domain-long-array' => 'prefixdefault-domain-long-array', + 'message-used-multiple-times' => 'prefixmessage-used-multiple-times', ], 'not_messages' => [ // translatable.html.php @@ -270,6 +271,7 @@ public function testExtraction($resource) 'typecast-long-array' => 'Available variables: a', 'default-domain-short-array' => null, 'default-domain-long-array' => null, + 'message-used-multiple-times' => 'Available variables: var1, var2, var3', ], 'not_messages' => [ // translatable.html.php diff --git a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php index 87a050a4a297f..7da213525f124 100644 --- a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php +++ b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php @@ -69,3 +69,8 @@ trans('default-domain-short-array', [], null); ?> trans('default-domain-long-array', array(), null); ?> + + +trans('message-used-multiple-times', ['var1' => 'val1', 'var2' => 'val2']); ?> +trans('message-used-multiple-times', ['var1' => 'val1', 'var2' => 'val2', 'var3' => 'val3']); ?> +trans('message-used-multiple-times', ['var1' => 'val1']); ?> From 3e0bb8ec1e4f4247704804f4c0a0ce445ec8ead5 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sat, 31 Oct 2020 23:37:20 +0100 Subject: [PATCH 14/23] Improving readability --- .../Translation/Catalogue/MergeOperation.php | 16 ++++++++++------ .../Translation/Catalogue/TargetOperation.php | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/Symfony/Component/Translation/Catalogue/MergeOperation.php b/src/Symfony/Component/Translation/Catalogue/MergeOperation.php index e10b68c4226df..02b25e51d08a4 100644 --- a/src/Symfony/Component/Translation/Catalogue/MergeOperation.php +++ b/src/Symfony/Component/Translation/Catalogue/MergeOperation.php @@ -37,20 +37,24 @@ protected function processDomain(string $domain) $intlDomain = $domain.MessageCatalogueInterface::INTL_DOMAIN_SUFFIX; foreach ($this->source->all($domain) as $id => $message) { + $sourceDomain = $this->source->defines($id, $intlDomain) ? $intlDomain : $domain; + $this->messages[$domain]['all'][$id] = $message; - $this->result->add([$id => $message], $this->source->defines($id, $intlDomain) ? $intlDomain : $domain); - if (null !== $keyMetadata = $this->source->getMetadata($id, $this->source->defines($id, $intlDomain) ? $intlDomain : $domain)) { - $this->result->setMetadata($id, $keyMetadata, $this->source->defines($id, $intlDomain) ? $intlDomain : $domain); + $this->result->add([$id => $message], $sourceDomain); + if (null !== $keyMetadata = $this->source->getMetadata($id, $sourceDomain)) { + $this->result->setMetadata($id, $keyMetadata, $sourceDomain); } } foreach ($this->target->all($domain) as $id => $message) { + $targetDomain = $this->target->defines($id, $intlDomain) ? $intlDomain : $domain; + if (!$this->source->has($id, $domain)) { $this->messages[$domain]['all'][$id] = $message; $this->messages[$domain]['new'][$id] = $message; - $this->result->add([$id => $message], $this->target->defines($id, $intlDomain) ? $intlDomain : $domain); - if (null !== $keyMetadata = $this->target->getMetadata($id, $this->target->defines($id, $intlDomain) ? $intlDomain : $domain)) { - $this->result->setMetadata($id, $keyMetadata, $this->target->defines($id, $intlDomain) ? $intlDomain : $domain); + $this->result->add([$id => $message], $targetDomain); + if (null !== $keyMetadata = $this->target->getMetadata($id, $targetDomain)) { + $this->result->setMetadata($id, $keyMetadata, $targetDomain); } } } diff --git a/src/Symfony/Component/Translation/Catalogue/TargetOperation.php b/src/Symfony/Component/Translation/Catalogue/TargetOperation.php index a17c05748b54d..7726a47860801 100644 --- a/src/Symfony/Component/Translation/Catalogue/TargetOperation.php +++ b/src/Symfony/Component/Translation/Catalogue/TargetOperation.php @@ -47,11 +47,13 @@ protected function processDomain(string $domain) // because doing so will not exclude messages like {x: x ∈ source ∧ x ∉ target.all ∧ x ∈ target.fallback} foreach ($this->source->all($domain) as $id => $message) { + $sourceDomain = $this->source->defines($id, $intlDomain) ? $intlDomain : $domain; + if ($this->target->has($id, $domain)) { $this->messages[$domain]['all'][$id] = $message; - $this->result->add([$id => $message], $this->source->defines($id, $intlDomain) ? $intlDomain : $domain); - if (null !== $keyMetadata = $this->source->getMetadata($id, $this->source->defines($id, $intlDomain) ? $intlDomain : $domain)) { - $this->result->setMetadata($id, $keyMetadata, $this->source->defines($id, $intlDomain) ? $intlDomain : $domain); + $this->result->add([$id => $message], $sourceDomain); + if (null !== $keyMetadata = $this->source->getMetadata($id, $sourceDomain)) { + $this->result->setMetadata($id, $keyMetadata, $sourceDomain); } } else { $this->messages[$domain]['obsolete'][$id] = $message; @@ -59,12 +61,14 @@ protected function processDomain(string $domain) } foreach ($this->target->all($domain) as $id => $message) { + $targetDomain = $this->target->defines($id, $intlDomain) ? $intlDomain : $domain; + if (!$this->source->has($id, $domain)) { $this->messages[$domain]['all'][$id] = $message; $this->messages[$domain]['new'][$id] = $message; - $this->result->add([$id => $message], $this->target->defines($id, $intlDomain) ? $intlDomain : $domain); - if (null !== $keyMetadata = $this->target->getMetadata($id, $this->target->defines($id, $intlDomain) ? $intlDomain : $domain)) { - $this->result->setMetadata($id, $keyMetadata, $this->target->defines($id, $intlDomain) ? $intlDomain : $domain); + $this->result->add([$id => $message], $targetDomain); + if (null !== $keyMetadata = $this->target->getMetadata($id, $targetDomain)) { + $this->result->setMetadata($id, $keyMetadata, $targetDomain); } } } From deb72d3c34bef6555871aa6627eb0bd80e2d902a Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sun, 1 Nov 2020 00:17:03 +0100 Subject: [PATCH 15/23] Always update variables with latest from code --- .../Catalogue/AbstractOperation.php | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php b/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php index 17c257fde458e..cd05e3a303e53 100644 --- a/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php +++ b/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php @@ -141,12 +141,57 @@ public function getResult() foreach ($this->getDomains() as $domain) { if (!isset($this->messages[$domain])) { $this->processDomain($domain); + $this->processVariablesMetadata($domain); } } return $this->result; } + /** + * Variables metadata should be updated according to target. + */ + protected function processVariablesMetadata(string $domain) + { + $intlDomain = $domain.MessageCatalogueInterface::INTL_DOMAIN_SUFFIX; + + foreach ($this->target->all($domain) as $id => $message) { + if ($this->source->has($id, $domain)) { + $targetDomain = $this->target->defines($id, $intlDomain) ? $intlDomain : $domain; + $sourceDomain = $this->source->defines($id, $intlDomain) ? $intlDomain : $domain; + $sourceMetadata = $this->result->getMetadata($id, $sourceDomain) ?? []; + $variablesNote = null; + + // Get target variables note + foreach ($this->target->getMetadata($id, $targetDomain)['notes'] ?? [] as $note) { + if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { + $variablesNote = $note; + + break; + } + } + + if ($variablesNote) { + // Update old variables note (if any) + if (isset($metadata['notes'])) { + foreach ($sourceMetadata['notes'] as $index => $note) { + if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { + $sourceMetadata['notes'][$index] = $variablesNote; + + break; + } + } + } else { + $sourceMetadata['notes'][] = $variablesNote; + } + + + $this->result->setMetadata($id, $sourceMetadata, $sourceDomain); + } + } + } + } + /** * Performs operation on source and target catalogues for the given domain and * stores the results. From e8cee751a96ded58306cbb5cfc643e95b5a8e014 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Fri, 13 Nov 2020 15:27:44 +0100 Subject: [PATCH 16/23] Fixing bug preventing update of variables from source in some cases --- .../Component/Translation/Catalogue/AbstractOperation.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php b/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php index cd05e3a303e53..3b3f0155a7f05 100644 --- a/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php +++ b/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php @@ -141,8 +141,9 @@ public function getResult() foreach ($this->getDomains() as $domain) { if (!isset($this->messages[$domain])) { $this->processDomain($domain); - $this->processVariablesMetadata($domain); } + + $this->processVariablesMetadata($domain); } return $this->result; @@ -173,7 +174,7 @@ protected function processVariablesMetadata(string $domain) if ($variablesNote) { // Update old variables note (if any) - if (isset($metadata['notes'])) { + if (isset($sourceMetadata['notes'])) { foreach ($sourceMetadata['notes'] as $index => $note) { if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { $sourceMetadata['notes'][$index] = $variablesNote; From 63a52b97ddfe2f6204c0e08361ad0623545a6219 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Fri, 13 Nov 2020 17:22:55 +0100 Subject: [PATCH 17/23] Fixing unintended CS errors (thanks @wouterj) --- src/Symfony/Bridge/Twig/Translation/TwigExtractor.php | 2 +- .../Component/Translation/Catalogue/AbstractOperation.php | 1 - src/Symfony/Component/Translation/Extractor/PhpExtractor.php | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php index ab2cece5de392..52d9b2071a225 100644 --- a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php +++ b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php @@ -95,7 +95,7 @@ protected function extractTemplate(string $template, MessageCatalogue $catalogue foreach ($metadata['notes'] as $index => $note) { if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { // Keep the higher variables count - if (count($extractedMessage[2]) > substr_count($note['content'], ',')) { + if (\count($extractedMessage[2]) > substr_count($note['content'], ',')) { $metadata['notes'][$index] = $variablesNote; } diff --git a/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php b/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php index 3b3f0155a7f05..cc74de467ff59 100644 --- a/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php +++ b/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php @@ -186,7 +186,6 @@ protected function processVariablesMetadata(string $domain) $sourceMetadata['notes'][] = $variablesNote; } - $this->result->setMetadata($id, $sourceMetadata, $sourceDomain); } } diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index 942251194b61a..55285795bcc87 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -361,7 +361,7 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string foreach ($metadata['notes'] as $index => $note) { if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { // Keep the higher variables count - if (count($variables) > substr_count($note['content'], ',')) { + if (\count($variables) > substr_count($note['content'], ',')) { $metadata['notes'][$index] = $variablesNote; } From 2a10f37454d93314bf58b9750b3ea63310591699 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Sat, 19 Dec 2020 00:24:10 +0100 Subject: [PATCH 18/23] Applying patch from fabbot (reviewed manually) --- .../Bridge/Twig/Tests/Translation/TwigExtractorTest.php | 4 ++-- .../Translation/Tests/fixtures/extractor/translation.html.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php index 873465b87ae2e..a7471846a5e4f 100644 --- a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php +++ b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php @@ -22,8 +22,8 @@ class TwigExtractorTest extends TestCase { - const VARIABLES_NOTE_CATEGORY = 'symfony-extractor-variables'; - const VARIABLES_NOTE_PREFIX = 'Available variables: '; + private const VARIABLES_NOTE_CATEGORY = 'symfony-extractor-variables'; + private const VARIABLES_NOTE_PREFIX = 'Available variables: '; /** * @dataProvider getExtractData diff --git a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php index 7da213525f124..2654ecaac209c 100644 --- a/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php +++ b/src/Symfony/Component/Translation/Tests/fixtures/extractor/translation.html.php @@ -70,7 +70,7 @@ trans('default-domain-long-array', array(), null); ?> - +Check behavior when the same key is used multiple times (no duplicate variables notes, keep higher variables count) trans('message-used-multiple-times', ['var1' => 'val1', 'var2' => 'val2']); ?> trans('message-used-multiple-times', ['var1' => 'val1', 'var2' => 'val2', 'var3' => 'val3']); ?> trans('message-used-multiple-times', ['var1' => 'val1']); ?> From 0a5366f55a1ddfe2536ddc4b8254e65b18fa36da Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Tue, 9 Mar 2021 12:54:32 +0100 Subject: [PATCH 19/23] Minor fixes --- .../Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php | 2 ++ src/Symfony/Bridge/Twig/Translation/TwigExtractor.php | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index 72fc5c5d3656a..11be192aaff3b 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -30,6 +30,7 @@ final class TranslationNodeVisitor extends AbstractNodeVisitor public const UNDEFINED_DOMAIN = '_undefined'; private $enabled = false; + /** * This class cannot read nodes backwards. * We need a way to track when we encounter a "|trans()" filter containing @@ -37,6 +38,7 @@ final class TranslationNodeVisitor extends AbstractNodeVisitor * as if it was used alone. */ private $skipTFunctionAfterFilter = false; + /** * This array stores found messages. * diff --git a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php index 52d9b2071a225..f5b005f058f4e 100644 --- a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php +++ b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php @@ -81,9 +81,10 @@ protected function extractTemplate(string $template, MessageCatalogue $catalogue $message = trim($extractedMessage[0]); $domain = $extractedMessage[1] ?: $this->defaultDomain; - $catalogue->set($message, $this->prefix.trim($extractedMessage[0]), $domain); + $catalogue->set($message, $this->prefix.$message, $domain); - if (!empty($extractedMessage[2])) { + // Are there any variables available for the current message? + if (\count($extractedMessage[2]) > 0) { $metadata = $catalogue->getMetadata($message, $domain); $variablesNote = [ 'category' => 'symfony-extractor-variables', From 46d26ec05131dd2d1ddaa76efb5f69be31786a1e Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Tue, 9 Mar 2021 12:55:28 +0100 Subject: [PATCH 20/23] Using constants for metadata key and prefix --- .../Tests/Translation/TwigExtractorTest.php | 9 +- .../Bridge/Twig/Translation/TwigExtractor.php | 8 +- .../Catalogue/AbstractOperation.php | 4 +- .../Translation/Extractor/PhpExtractor.php | 8 +- .../Translation/MessageCatalogue.php | 12 +++ .../Tests/Extractor/PhpExtractorTest.php | 102 +++++++++--------- 6 files changed, 76 insertions(+), 67 deletions(-) diff --git a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php index a7471846a5e4f..3494d6d17b0ed 100644 --- a/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php +++ b/src/Symfony/Bridge/Twig/Tests/Translation/TwigExtractorTest.php @@ -22,9 +22,6 @@ class TwigExtractorTest extends TestCase { - private const VARIABLES_NOTE_CATEGORY = 'symfony-extractor-variables'; - private const VARIABLES_NOTE_PREFIX = 'Available variables: '; - /** * @dataProvider getExtractData */ @@ -58,8 +55,8 @@ public function testExtract($template, $messages) // Check variables if ($notes = $catalogue->getMetadata($key, $data[0])['notes'] ?? null) { foreach ($notes as $note) { - if (isset($note['category']) && self::VARIABLES_NOTE_CATEGORY === $note['category']) { - $this->assertEquals(self::VARIABLES_NOTE_PREFIX.$data[1], $note['content']); + if (isset($note['category']) && MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY === $note['category']) { + $this->assertEquals(MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.$data[1], $note['content']); break; } @@ -76,7 +73,7 @@ public function getExtractData() * 1 => [ * 'message_key' => [ * 0 => 'domain', - * 1 => 'var1, var2'|null, // Complete message is 'Available variables: var1, var2' + * 1 => 'var1, var2'|null, // Complete message is 'Variables: var1, var2' * ... * ] * ], diff --git a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php index f5b005f058f4e..0c42ef454c3c1 100644 --- a/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php +++ b/src/Symfony/Bridge/Twig/Translation/TwigExtractor.php @@ -87,15 +87,15 @@ protected function extractTemplate(string $template, MessageCatalogue $catalogue if (\count($extractedMessage[2]) > 0) { $metadata = $catalogue->getMetadata($message, $domain); $variablesNote = [ - 'category' => 'symfony-extractor-variables', - 'content' => 'Available variables: '.implode(', ', $extractedMessage[2]), + 'category' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY, + 'content' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.implode(', ', $extractedMessage[2]), ]; // Update old variables note (if any) if (isset($metadata['notes'])) { foreach ($metadata['notes'] as $index => $note) { - if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { - // Keep the higher variables count + if (isset($note['category']) && MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY === $note['category']) { + // Keep the highest variables count if (\count($extractedMessage[2]) > substr_count($note['content'], ',')) { $metadata['notes'][$index] = $variablesNote; } diff --git a/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php b/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php index cc74de467ff59..99398f1790dc7 100644 --- a/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php +++ b/src/Symfony/Component/Translation/Catalogue/AbstractOperation.php @@ -165,7 +165,7 @@ protected function processVariablesMetadata(string $domain) // Get target variables note foreach ($this->target->getMetadata($id, $targetDomain)['notes'] ?? [] as $note) { - if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { + if (isset($note['category']) && MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY === $note['category']) { $variablesNote = $note; break; @@ -176,7 +176,7 @@ protected function processVariablesMetadata(string $domain) // Update old variables note (if any) if (isset($sourceMetadata['notes'])) { foreach ($sourceMetadata['notes'] as $index => $note) { - if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { + if (isset($note['category']) && MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY === $note['category']) { $sourceMetadata['notes'][$index] = $variablesNote; break; diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index 55285795bcc87..349b4bce44795 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -352,15 +352,15 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string if (!empty($variables)) { $variablesNote = [ - 'category' => 'symfony-extractor-variables', - 'content' => 'Available variables: '.implode(', ', $variables), + 'category' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY, + 'content' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.implode(', ', $variables), ]; // Update old variables note (if any) if (isset($metadata['notes'])) { foreach ($metadata['notes'] as $index => $note) { - if (isset($note['category']) && 'symfony-extractor-variables' === $note['category']) { - // Keep the higher variables count + if (isset($note['category']) && MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY === $note['category']) { + // Keep the highest variables count if (\count($variables) > substr_count($note['content'], ',')) { $metadata['notes'][$index] = $variablesNote; } diff --git a/src/Symfony/Component/Translation/MessageCatalogue.php b/src/Symfony/Component/Translation/MessageCatalogue.php index d50ae03e97bbc..ed33fc6d036ea 100644 --- a/src/Symfony/Component/Translation/MessageCatalogue.php +++ b/src/Symfony/Component/Translation/MessageCatalogue.php @@ -19,6 +19,18 @@ */ class MessageCatalogue implements MessageCatalogueInterface, MetadataAwareInterface { + /** + * This metadata key is used to store a note containing available variables for each message + */ + public const METADATA_AVAILABLE_VARIABLES_KEY = 'symfony-extractor-variables'; + + /** + * The comma-separated list of available variables is appended to this prefix. + * + * Example: "Variables: foo, bar" + */ + public const METADATA_AVAILABLE_VARIABLES_PREFIX = 'Variables: '; + private $messages = []; private $metadata = []; private $resources = []; diff --git a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php index 8a418775886b0..075892761c936 100644 --- a/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php +++ b/src/Symfony/Component/Translation/Tests/Extractor/PhpExtractorTest.php @@ -189,14 +189,14 @@ public function testExtraction($resource) 'translatable concatenated message with heredoc and nowdoc' => null, 'translatable test-no-params-short-array' => null, 'translatable test-no-params-long-array' => null, - 'translatable test-params-short-array' => 'Available variables: foo', - 'translatable test-params-long-array' => 'Available variables: foo', - 'translatable test-multiple-params-short-array' => 'Available variables: foo, foz', - 'translatable test-multiple-params-long-array' => 'Available variables: foo, foz', - 'translatable test-params-trailing-comma-short-array' => 'Available variables: foo', - 'translatable test-params-trailing-comma-long-array' => 'Available variables: foo', - 'translatable typecast-short-array' => 'Available variables: a', - 'translatable typecast-long-array' => 'Available variables: a', + 'translatable test-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable test-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable test-multiple-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo, foz', + 'translatable test-multiple-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo, foz', + 'translatable test-params-trailing-comma-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable test-params-trailing-comma-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable typecast-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', + 'translatable typecast-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', 'translatable default-domain-short-array' => null, 'translatable default-domain-long-array' => null, @@ -213,14 +213,14 @@ public function testExtraction($resource) 'translatable-fqn concatenated message with heredoc and nowdoc' => null, 'translatable-fqn test-no-params-short-array' => null, 'translatable-fqn test-no-params-long-array' => null, - 'translatable-fqn test-params-short-array' => 'Available variables: foo', - 'translatable-fqn test-params-long-array' => 'Available variables: foo', - 'translatable-fqn test-multiple-params-short-array' => 'Available variables: foo, foz', - 'translatable-fqn test-multiple-params-long-array' => 'Available variables: foo, foz', - 'translatable-fqn test-params-trailing-comma-short-array' => 'Available variables: foo', - 'translatable-fqn test-params-trailing-comma-long-array' => 'Available variables: foo', - 'translatable-fqn typecast-short-array' => 'Available variables: a', - 'translatable-fqn typecast-long-array' => 'Available variables: a', + 'translatable-fqn test-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-fqn test-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-fqn test-multiple-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo, foz', + 'translatable-fqn test-multiple-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo, foz', + 'translatable-fqn test-params-trailing-comma-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-fqn test-params-trailing-comma-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-fqn typecast-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', + 'translatable-fqn typecast-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', 'translatable-fqn default-domain-short-array' => null, 'translatable-fqn default-domain-long-array' => null, @@ -237,14 +237,14 @@ public function testExtraction($resource) 'translatable-short concatenated message with heredoc and nowdoc' => null, 'translatable-short test-no-params-short-array' => null, 'translatable-short test-no-params-long-array' => null, - 'translatable-short test-params-short-array' => 'Available variables: foo', - 'translatable-short test-params-long-array' => 'Available variables: foo', - 'translatable-short test-multiple-params-short-array' => 'Available variables: foo, foz', - 'translatable-short test-multiple-params-long-array' => 'Available variables: foo, foz', - 'translatable-short test-params-trailing-comma-short-array' => 'Available variables: foo', - 'translatable-short test-params-trailing-comma-long-array' => 'Available variables: foo', - 'translatable-short typecast-short-array' => 'Available variables: a', - 'translatable-short typecast-long-array' => 'Available variables: a', + 'translatable-short test-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-short test-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-short test-multiple-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo, foz', + 'translatable-short test-multiple-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo, foz', + 'translatable-short test-params-trailing-comma-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-short test-params-trailing-comma-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-short typecast-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', + 'translatable-short typecast-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', 'translatable-short default-domain-short-array' => null, 'translatable-short default-domain-long-array' => null, @@ -261,50 +261,50 @@ public function testExtraction($resource) 'concatenated message with heredoc and nowdoc' => null, 'test-no-params-short-array' => null, 'test-no-params-long-array' => null, - 'test-params-short-array' => 'Available variables: foo', - 'test-params-long-array' => 'Available variables: foo', - 'test-multiple-params-short-array' => 'Available variables: foo, foz', - 'test-multiple-params-long-array' => 'Available variables: foo, foz', - 'test-params-trailing-comma-short-array' => 'Available variables: foo', - 'test-params-trailing-comma-long-array' => 'Available variables: foo', - 'typecast-short-array' => 'Available variables: a', - 'typecast-long-array' => 'Available variables: a', + 'test-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'test-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'test-multiple-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo, foz', + 'test-multiple-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo, foz', + 'test-params-trailing-comma-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'test-params-trailing-comma-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'typecast-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', + 'typecast-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', 'default-domain-short-array' => null, 'default-domain-long-array' => null, - 'message-used-multiple-times' => 'Available variables: var1, var2, var3', + 'message-used-multiple-times' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'var1, var2, var3', ], 'not_messages' => [ // translatable.html.php 'translatable other-domain-test-no-params-short-array' => null, 'translatable other-domain-test-no-params-long-array' => null, - 'translatable other-domain-test-params-short-array' => 'Available variables: foo', - 'translatable other-domain-test-params-long-array' => 'Available variables: foo', - 'translatable other-domain-typecast-short-array' => 'Available variables: a', - 'translatable other-domain-typecast-long-array' => 'Available variables: a', + 'translatable other-domain-test-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable other-domain-test-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable other-domain-typecast-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', + 'translatable other-domain-typecast-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', // translatable-fqn.html.php 'translatable-fqn other-domain-test-no-params-short-array' => null, 'translatable-fqn other-domain-test-no-params-long-array' => null, - 'translatable-fqn other-domain-test-params-short-array' => 'Available variables: foo', - 'translatable-fqn other-domain-test-params-long-array' => 'Available variables: foo', - 'translatable-fqn other-domain-typecast-short-array' => 'Available variables: a', - 'translatable-fqn other-domain-typecast-long-array' => 'Available variables: a', + 'translatable-fqn other-domain-test-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-fqn other-domain-test-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-fqn other-domain-typecast-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', + 'translatable-fqn other-domain-typecast-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', // translatable-short.html.php 'translatable-short other-domain-test-no-params-short-array' => null, 'translatable-short other-domain-test-no-params-long-array' => null, - 'translatable-short other-domain-test-params-short-array' => 'Available variables: foo', - 'translatable-short other-domain-test-params-long-array' => 'Available variables: foo', - 'translatable-short other-domain-typecast-short-array' => 'Available variables: a', - 'translatable-short other-domain-typecast-long-array' => 'Available variables: a', + 'translatable-short other-domain-test-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-short other-domain-test-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'translatable-short other-domain-typecast-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', + 'translatable-short other-domain-typecast-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', // translation.html.php 'other-domain-test-no-params-short-array' => null, 'other-domain-test-no-params-long-array' => null, - 'other-domain-test-params-short-array' => 'Available variables: foo', - 'other-domain-test-params-long-array' => 'Available variables: foo', - 'other-domain-typecast-short-array' => 'Available variables: a', - 'other-domain-typecast-long-array' => 'Available variables: a', + 'other-domain-test-params-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'other-domain-test-params-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'foo', + 'other-domain-typecast-short-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', + 'other-domain-typecast-long-array' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.'a', ], ]; $actualCatalogue = $catalogue->all(); @@ -390,13 +390,13 @@ private function getVariablesNoteContentFromMetadata(array $metadata) * 'notes' => [ * 0 => [ * 'category' => 'symfony-extractor-variables', - * 'content' => 'Available variables: var1, var2', + * 'content' => 'Available: var1, var2', * ], * ... * ], * ... * ] */ - return array_filter($metadata['notes'] ?? [], function ($note) { return 'symfony-extractor-variables' === $note['category']; })[0]['content'] ?? null; + return array_filter($metadata['notes'] ?? [], function ($note) { return MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY === $note['category']; })[0]['content'] ?? null; } } From 1612c085f7f6c7c70ff38e6fd20c076729a10119 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Tue, 9 Mar 2021 13:09:12 +0100 Subject: [PATCH 21/23] Fixing CI --- .../Twig/NodeVisitor/TranslationNodeVisitor.php | 2 +- .../Command/TranslationUpdateCommand.php | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index 11be192aaff3b..c067eb7cc8481 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -208,7 +208,7 @@ private function getReadVariablesFromArguments(Node $arguments, int $index): arr return $this->getReadVariablesFromNode($argument); } - private function getReadVariablesFromNode(Node $node): ?array + private function getReadVariablesFromNode(Node $node): array { if (!empty($node)) { $variables = []; diff --git a/src/Symfony/Bundle/FrameworkBundle/Command/TranslationUpdateCommand.php b/src/Symfony/Bundle/FrameworkBundle/Command/TranslationUpdateCommand.php index 03d86191b07b5..924e028b22515 100644 --- a/src/Symfony/Bundle/FrameworkBundle/Command/TranslationUpdateCommand.php +++ b/src/Symfony/Bundle/FrameworkBundle/Command/TranslationUpdateCommand.php @@ -24,6 +24,7 @@ use Symfony\Component\Translation\Extractor\ExtractorInterface; use Symfony\Component\Translation\MessageCatalogue; use Symfony\Component\Translation\MessageCatalogueInterface; +use Symfony\Component\Translation\MetadataAwareInterface; use Symfony\Component\Translation\Reader\TranslationReaderInterface; use Symfony\Component\Translation\Writer\TranslationWriterInterface; @@ -242,10 +243,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int $result->replace($allIntlMessages + $newMessages, $intlDomain); // Move new metadata - foreach ($newMessages as $key => $message) { - if (null !== $extractedCatalogue->getMetadata($key, $domain)) { - $result->setMetadata($key, $extractedCatalogue->getMetadata($key, $domain), $intlDomain); - $extractedCatalogue->deleteMetadata($key, $domain); + if ($result instanceof MetadataAwareInterface) { + foreach ($newMessages as $key => $message) { + if (null !== $extractedCatalogue->getMetadata($key, $domain)) { + $result->setMetadata($key, $extractedCatalogue->getMetadata($key, $domain), $intlDomain); + $extractedCatalogue->deleteMetadata($key, $domain); + } } } } From 8ec6086f697d69e0d61b2987afa234cbc92b7bf0 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Tue, 9 Mar 2021 15:37:09 +0100 Subject: [PATCH 22/23] Improving code quality --- .../NodeVisitor/TranslationNodeVisitor.php | 32 +++++++++++++------ .../Translation/Extractor/PhpExtractor.php | 2 +- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index c067eb7cc8481..8333f80b4b21b 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -195,6 +195,10 @@ private function getReadMessageFromNode(Node $node): ?string return null; } + /** + * Extracts variable names from a @Node containing all arguments passed to + * a Twig function/filter. + */ private function getReadVariablesFromArguments(Node $arguments, int $index): array { if ($arguments->hasNode('vars')) { @@ -208,24 +212,32 @@ private function getReadVariablesFromArguments(Node $arguments, int $index): arr return $this->getReadVariablesFromNode($argument); } + /** + * Extracts variable names from a @Node representing the array of + * parameters passed to the translation function/filter. + */ private function getReadVariablesFromNode(Node $node): array { - if (!empty($node)) { - $variables = []; + if (\count($node) <= 0) { + return []; + } - foreach ($node as $key => $variable) { - // Odd children are variable names, even ones are values - if (1 == $key % 2) { - continue; - } + $variables = []; + $isVariableName = true; - $variables[] = $variable->getAttribute('value'); + foreach ($node as $parameterToken) { + /* + * Variable names and values are direct children of the parent node (e.g. ['var1', 'value1', 'var2', + * 'value2', 'var3', 'value3']), so we can get all names by skipping the values every two cycles. + */ + if ($isVariableName ^= 1) { + continue; } - return $variables; + $variables[] = $parameterToken->getAttribute('value'); } - return []; + return $variables; } private function getReadDomainFromArguments(Node $arguments, int $index): ?string diff --git a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php index 349b4bce44795..635c64be7ee7f 100644 --- a/src/Symfony/Component/Translation/Extractor/PhpExtractor.php +++ b/src/Symfony/Component/Translation/Extractor/PhpExtractor.php @@ -350,7 +350,7 @@ protected function parseTokens(array $tokens, MessageCatalogue $catalog, string $normalizedFilename = preg_replace('{[\\\\/]+}', '/', $filename); $metadata['sources'][] = $normalizedFilename.':'.$tokens[$key][2]; - if (!empty($variables)) { + if (\count($variables) > 0) { $variablesNote = [ 'category' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_KEY, 'content' => MessageCatalogue::METADATA_AVAILABLE_VARIABLES_PREFIX.implode(', ', $variables), From 0025c52f418d0a74ca70b7f869bcc97555f4ea04 Mon Sep 17 00:00:00 2001 From: Marco Lipparini Date: Mon, 15 Mar 2021 13:58:07 +0100 Subject: [PATCH 23/23] Fixing static analysis (Psalm) --- .../Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php index 8333f80b4b21b..a5211844dc163 100644 --- a/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php +++ b/src/Symfony/Bridge/Twig/NodeVisitor/TranslationNodeVisitor.php @@ -11,6 +11,7 @@ namespace Symfony\Bridge\Twig\NodeVisitor; +use ArrayIterator; use Symfony\Bridge\Twig\Node\TransNode; use Twig\Environment; use Twig\Node\Expression\Binary\ConcatBinary; @@ -104,8 +105,10 @@ protected function doEnterNode(Node $node, Environment $env): Node ) { // extract t() nodes with a trans filter applied $functionNodeArguments = $node->getNode('node')->getNode('arguments'); + /** @var ArrayIterator $iterator */ + $iterator = $functionNodeArguments->getIterator(); - if ($functionNodeArguments->getIterator()->current() instanceof ConstantExpression) { + if ($iterator->current() instanceof ConstantExpression) { $this->messages[] = [ $this->getReadMessageFromArguments($functionNodeArguments, 0), $this->getReadDomainFromArguments($functionNodeArguments, 2), @@ -127,8 +130,10 @@ protected function doEnterNode(Node $node, Environment $env): Node } $functionNodeArguments = $node->getNode('arguments'); + /** @var ArrayIterator $iterator */ + $iterator = $functionNodeArguments->getIterator(); - if ($functionNodeArguments->getIterator()->current() instanceof ConstantExpression) { + if ($iterator->current() instanceof ConstantExpression) { $this->messages[] = [ $this->getReadMessageFromArguments($functionNodeArguments, 0), $this->getReadDomainFromArguments($functionNodeArguments, 2),