8000 feature #46161 [Translation] Add `PhpAstExtractor` (welcoMattic) · symfony/symfony@38678dc · GitHub
[go: up one dir, main page]
Skip to content
8000

Commit 38678dc

Browse files
fabpotfabpot
committed
feature #46161 [Translation] Add PhpAstExtractor (welcoMattic)
This PR was merged into the 6.2 branch. Discussion ---------- [Translation] Add `PhpAstExtractor` | Q | A | ------------- | --- | Branch? | 6.2 | Bug fix? | no | New feature? | yes | Deprecations? | no | Tickets | Fix #44899, #42285, #45972, #45039 | License | MIT | Doc PR | symfony/symfony-docs#17369 After discussions with `@stof` and `@nicolas`-grekas at SymfonyLive Paris 2022, it appears clear that we need a brand new PhpExtractor in Translation to support various syntax, especially PHP8 new syntax (named arguments for example). In order to make this new extractor sustainable, performant and maintainable, it must be based on AST. That's why this PR adds a new optional dependency on [nikic/php-parser](https://github.com/nikic/PHP-Parser). The tests suite is the same as PhpExtractorTest, in addition to new cases for newly supported syntax. NB: I was inspired by https://github.com/php-translation/extractor design, with adaptations for Symfony. To-do: - [x] As suggested by `@Nyholm`, take a look at https://github.com/php-translation/extractor - [x] Fix #45039 (could be achieved easily with AST tree) Commits ------- 5d4a81f Add PhpAstExtractor
2 parents 0de91ed + 5d4a81f commit 38678dc

24 files changed

+974
-0
lines changed

‎UPGRADE-6.2.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,12 @@ Serializer
101101
* Change the signature of `AttributeMetadataInterface::setSerializedName()` to `setSerializedName(?string)`
102102
* Change the signature of `ClassMetadataInterface::setClassDiscriminatorMapping()` to `setClassDiscriminatorMapping(?ClassDiscriminatorMapping)`
103103

104+
Translation
105+
-----------
106+
107+
* Deprecate `PhpExtractor` in favor of `PhpAstExtractor`
108+
* Add `PhpAstExtractor` (requires [nikic/php-parser](https://github.com/nikic/php-parser) to be installed)
109+
104110
Validator
105111
---------
106112

‎src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Compiler/UnusedTagsPass.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ class UnusedTagsPass implements CompilerPassInterface
8989
'texter.transport_factory',
9090
'translation.dumper',
9191
'translation.extractor',
92+
'translation.extractor.visitor',
9293
'translation.loader',
9394
'translation.provider_factory',
9495
'twig.extension',

‎src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
use Http\Client\HttpClient;
1818
use phpDocumentor\Reflection\DocBlockFactoryInterface;
1919
use phpDocumentor\Reflection\Types\ContextFactory;
20+
use PhpParser\Parser;
2021
use PHPStan\PhpDocParser\Parser\PhpDocParser;
2122
use Psr\Cache\CacheItemPoolInterface;
2223
use Psr\Container\ContainerInterface as PsrContainerInterface;
@@ -216,6 +217,7 @@
216217
use Symfony\Component\Translation\Bridge\Loco\LocoProviderFactory;
217218
use Symfony\Component\Translation\Bridge\Lokalise\LokaliseProviderFactory;
218219
use Symfony\Component\Translation\Command\XliffLintCommand as BaseXliffLintCommand;
220+
use Symfony\Component\Translation\Extractor\PhpAstExtractor;
219221
use Symfony\Component\Translation\LocaleSwitcher;
220222
use Symfony\Component\Translation\PseudoLocalizationTranslator;
221223
use Symfony\Component\Translation\Translator;
@@ -1313,6 +1315,14 @@ private function registerTranslatorConfiguration(array $config, ContainerBuilder
13131315
$container->removeDefinition('translation.locale_switcher');
13141316
}
13151317

1318+
if (ContainerBuilder::willBeAvailable('nikic/php-parser', Parser::class, ['symfony/translation'])
1319+
&& ContainerBuilder::willBeAvailable('symfony/translation', PhpAstExtractor::class, ['symfony/framework-bundle'])
1320+
) {
1321+
$container->removeDefinition('translation.extractor.php');
1322+
} else {
1323+
$container->removeDefinition('translation.extractor.php_ast');
1324+
}
1325+
13161326
$loader->load('translation_providers.php');
13171327

13181328
// Use the "real" translator instead of the identity default

‎src/Symfony/Bundle/FrameworkBundle/Resources/config/translation.php

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@
2626
use Symfony\Component\Translation\Dumper\YamlFileDumper;
2727
use Symfony\Component\Translation\Extractor\ChainExtractor;
2828
use Symfony\Component\Translation\Extractor\ExtractorInterface;
29+
use Symfony\Component\Translation\Extractor\PhpAstExtractor;
2930
use Symfony\Component\Translation\Extractor\PhpExtractor;
31+
use Symfony\Component\Translation\Extractor\Visitor\ConstraintVisitor;
32+
use Symfony\Component\Translation\Extractor\Visitor\TranslatableMessageVisitor;
33+
use Symfony\Component\Translation\Extractor\Visitor\TransMethodVisitor;
3034
use Symfony\Component\Translation\Formatter\MessageFormatter;
3135
use Symfony\Component\Translation\Loader\CsvFileLoader;
3236
use Symfony\Component\Translation\Loader\IcuDatFileLoader;
@@ -151,6 +155,19 @@
151155
->set('translation.extractor.php', PhpExtractor::class)
152156
->tag('translation.extractor', ['alias' => 'php'])
153157

158+
->set('translation.extractor.php_ast', PhpAstExtractor::class)
159+
->args([tagged_iterator('translation.extractor.visitor')])
160+
->tag('translation.extractor', ['alias' => 'php'])
161+
162+
->set('translation.extractor.visitor.trans_method', TransMethodVisitor::class)
163+
->tag('translation.extractor.visitor')
164+
165+
->set('translation.extractor.visitor.translatable_message', TranslatableMessageVisitor::class)
166+
->tag('translation.extractor.visitor')
167+
168+
->set('translation.extractor.visitor.constraint', ConstraintVisitor::class)
169+
->tag('translation.extractor.visitor')
170+
154171
->set('translation.reader', TranslationReader::class)
155172
->alias(TranslationReaderInterface::class, 'translation.reader')
156173

‎src/Symfony/Component/Translation/CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
CHANGELOG
22
=========
33

4+
6.2
5+
---
6+
7+
* Deprecate `PhpExtractor` in favor of `PhpAstExtractor`
8+
* Add `PhpAstExtractor` (requires [nikic/php-parser](https://github.com/nikic/php-parser) to be installed)
9+
410
6.1
511
---
612

‎src/Symfony/Component/Translation/DependencyInjection/TranslatorPass.php

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,19 @@ public function process(ContainerBuilder $container)
4949
->replaceArgument(3, $loaders)
5050
;
5151

52+
if ($container->hasDefinition('validator') && $container->hasDefinition('translation.extractor.visitor.constraint')) {
53+
$constraintVisitorDefinition = $container->getDefinition('translation.extractor.visitor.constraint');
54+
$constraintClassNames = [];
55+
56+
foreach ($container->findTaggedServiceIds('validator.constraint_validator', true) as $id => $attributes) {
57+
$serviceDefinition = $container->getDefinition($id);
58+
// Extraction of the constraint class name from the Constraint Validator FQCN
59+
$constraintClassNames[] = str_replace('Validator', '', substr(strrchr($serviceDefinition->getClass(), '\\'), 1));
60+
}
61+
62+
$constraintVisitorDefinition->setArgument(0, $constraintClassNames);
63+
}
64+
5265
if (!$container->hasParameter('twig.default_path')) {
5366
return;
5467
}

‎src/Symfony/Component/Translation/Extractor/PhpAstExtractor.php

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\Component\Translation\Extractor;
13+
14+
use PhpParser\NodeTraverser;
15+
use PhpParser\NodeVisitor;
16+
use PhpParser\Parser;
17+
use PhpParser\ParserFactory;
18+
use Symfony\Component\Finder\Finder;
19+
use Symfony\Component\Translation\Extractor\Visitor\AbstractVisitor;
20+
use Symfony\Component\Translation\MessageCatalogue;
21+
22+
/**
23+
* PhpAstExtractor extracts translation messages from a PHP AST.
24+
*
25+
* @author Mathieu Santostefano <msantostefano@protonmail.com>
26+
*/
27+
final class PhpAstExtractor extends AbstractFileExtractor implements ExtractorInterface
28+
{
29+
private Parser $parser;
30+
31+
public function __construct(
32+
/**
33+
* @param iterable<AbstractVisitor&NodeVisitor> $visitors
34+
*/
35+
private readonly iterable $visitors,
36+
private string $prefix = '',
37+
) {
38+
if (!class_exists(ParserFactory::class)) {
39+
throw new \LogicException(sprintf('You cannot use "%s" as the "nikic/php-parser" package is not installed. Try running "composer require nikic/php-parser".', static::class));
40+
}
41+
42+
$this->parser = (new ParserFactory())->create(ParserFactory::PREFER_PHP7);
43+
}
44+
45+
public function extract(iterable|string $resource, MessageCatalogue $catalogue): void
46+
{
47+
foreach ($this->extractFiles($resource) as $file) {
48+
$traverser = new NodeTraverser();
49+
/** @var AbstractVisitor&NodeVisitor $visitor */
50+
foreach ($this->visitors as $visitor) {
51+
$visitor->initialize($catalogue, $file, $this->prefix);
52+
$traverser->addVisitor($visitor);
53+
}
54+
55+
$nodes = $this->parser->parse(file_get_contents($file));
56+
$traverser->traverse($nodes);
57+
}
58+
}
59+
60+
public function setPrefix(string $prefix): void
61+
{
62+
$this->prefix = $prefix;
63+
}
64+
65+
protected function canBeExtracted(string $file): bool
66+
{
67+
return 'php' === pathinfo($file, \PATHINFO_EXTENSION) && $this->isFile($file);
68+
}
69+
70+
protected function extractFromDirectory(array|string $resource): iterable|Finder
71+
{
72+
if (!class_exists(Finder::class)) {
73+
throw new \LogicException(sprintf('You cannot use "%s" as the "symfony/finder" package is not installed. Try running "composer require symfony/finder".', static::class));
74+
}
75+
76+
return (new Finder())->files()->name('*.php')->in($resource);
77+
}
78+
}

‎src/Symfony/Component/Translation/Extractor/PhpExtractor.php

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,17 @@
1111

1212
namespace Symfony\Component\Translation\Extractor;
1313

14+
trigger_deprecation('symfony/translation', '6.2', '"%s" is deprecated, use "%s" instead.', PhpExtractor::class, PhpAstExtractor::class);
15+
1416
use Symfony\Component\Finder\Finder;
1517
use Symfony\Component\Translation\MessageCatalogue;
1618

1719
/**
1820
* PhpExtractor extracts translation messages from a PHP template.
1921
*
2022
* @author Michel Salib <michelsalib@hotmail.com>
23+
*
24+
* @deprecated since Symfony 6.2, use the PhpAstExtractor instead
2125
*/
2226
class PhpExtractor extends AbstractFileExtractor implements ExtractorInterface
2327
{

‎src/Symfony/Component/Translation/Extractor/Visitor/AbstractVisitor.php

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\Component\Translation\Extractor\Visitor;
13+
14+
use PhpParser\Node;
15+
use Symfony\Component\Translation\MessageCatalogue;
16+
17+
/**
18+
* @author Mathieu Santostefano <msantostefano@protonmail.com>
19+
*/
20+
abstract class AbstractVisitor
21+
{
22+
private MessageCatalogue $catalogue;
23+
private \SplFileInfo $file;
24+
private string $messagePrefix;
25+
26+
public function initialize(MessageCatalogue $catalogue, \SplFileInfo $file, string $messagePrefix): void
27+
{
28+
$this->catalogue = $catalogue;
29+
$this->file = $file;
30+
$this->messagePrefix = $messagePrefix;
31+
}
32+
33+
protected function addMessageToCatalogue(string $message, ?string $domain, int $line): void
34+
{
35+
$domain ??= 'messages';
36+
$this->catalogue->set($message, $this->messagePrefix.$message, $domain);
37+
$metadata = $this->catalogue->getMetadata($message, $domain) ?? [];
38+
$normalizedFilename = preg_replace('{[\\\\/]+}', '/', $this->file);
39+
$metadata['sources'][] = $normalizedFilename.':'.$line;
40+
$this->catalogue->setMetadata($message, $metadata, $domain);
41+
}
42+
43+
protected function getStringArguments(Node\Expr\CallLike|Node\Attribute|Node\Expr\New_ $node, int|string $index, bool $indexIsRegex = false): array
44+
{
45+
$args = $node instanceof Node\Expr\CallLike ? $node->getArgs() : $node->args;
46+
47+
if (\is_string($index)) {
48+
return $this->getStringNamedArguments($node, $index, $indexIsRegex);
49+
}
50+
51+
if (\count($args) < $index) {
52+
return [];
53+
}
54+
55+
/** @var Node\Arg $arg */
56+
$arg = $args[$index];
57+
if (!$result = $this->getStringValue($arg->value)) {
58+
return [];
59+
}
60+
61+
return [$result];
62+
}
63+
64+
protected function hasNodeNamedArguments(Node\Expr\CallLike|Node\Attribute|Node\Expr\New_ $node): bool
65+
{
66+
$args = $node instanceof Node\Expr\CallLike ? $node->getArgs() : $node->args;
67+
68+
/** @var Node\Arg $arg */
69+
foreach ($args as $arg) {
70+
if (null !== $arg->name) {
71+
return true;
72+
}
73+
}
74+
75+
return false;
76+
}
77+
78+
private function getStringNamedArguments(Node\Expr\CallLike|Node\Attribute $node, string $argumentName = null, bool $isArgumentNamePattern = false): array
79+
{
80+
$args = $node instanceof Node\Expr\CallLike ? $node->getArgs() : $node->args;
81+
$argumentValues = [];
82+
83+
foreach ($args as $arg) {
84+
if (!$isArgumentNamePattern && $arg->name?->toString() === $argumentName) {
85+
$argumentValues[] = $this->getStringValue($arg->value);
86+
} elseif ($isArgumentNamePattern && preg_match($argumentName, $arg->name?->toString() ?? '') > 0) {
87+
$argumentValues[] = $this->getStringValue($arg->value);
88+
}
89+
}
90+
91+
return array_filter($argumentValues);
92+
}
93+
94+
private function getStringValue(Node $node): ?string
95+
{
96+
if ($node instanceof Node\Scalar\String_) {
97+
return $node->value;
98+
}
99+
100+
if ($node instanceof Node\Expr\BinaryOp\Concat) {
101+
if (null === $left = $this->getStringValue($node->left)) {
102+
return null;
103+
}
104+
105+
if (null === $right = $this->getStringValue($node->right)) {
106+
return null;
107+
}
108+
109+
return $left.$right;
110+
}
111+
112+
if ($node instanceof Node\Expr\Assign && $node->expr instanceof Node\Scalar\String_) {
113+
return $node->expr->value;
114+
}
115+
116+
return null;
117+
}
118+
}

0 commit comments

Comments
 (0)
0