8000 [Translation] Add icu message support for the lokalise provider by DanielBadura · Pull Request #45368 · symfony/symfony · GitHub
[go: up one dir, main page]

Skip to content

[Translation] Add icu message support for the lokalise provider #45368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: 5.4
Choose a base branch
from
Original file line number Diff line number Diff line change
Expand Up @@ -857,6 +857,7 @@ private function addTranslatorSection(ArrayNodeDefinition $rootNode, callable $e
->fixXmlConfig('locale')
->children()
->scalarNode('dsn')->end()
->booleanNode('intl_icu_enabled')->defaultFalse()->end()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
->booleanNode('intl_icu_enabled')->defaultFalse()->end()
->booleanNode('intl_icu')->defaultFalse()->end()

I think it's better to name it short.

->arrayNode('domains')
->prototype('scalar')->end()
->defaultValue([])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1476,6 +1476,8 @@ private function registerTranslatorConfiguration(array $config, ContainerBuilder
;

$container->getDefinition('translation.provider_collection')->setArgument(0, $config['providers']);

$container->getDefinition('translation.provider_factory.lokalise')->setArgument(4, $config['providers']['lokalise']['intl_icu_enabled']);
}

private function registerValidationConfiguration(array $config, ContainerBuilder $container, PhpFileLoader $loader, bool $propertyInfoEnabled)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

class CrowdinProviderTest extends ProviderTestCase
{
public function createProvider(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint): ProviderInterface
public function createProvider(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint, bool $intlIcuEnabled = false): ProviderInterface
{
return new CrowdinProvider($client, $loader, $logger, $this->getXliffFileDumper(), $defaultLocale, $endpoint);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

class LocoProviderTest extends ProviderTestCase
{
public function createProvider(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint): ProviderInterface
public function createProvider(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint, bool $intlIcuEnabled = false): ProviderInterface
{
return new LocoProvider($client, $loader, $logger, $defaultLocale, $endpoint);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,16 @@ final class LokaliseProvider implements ProviderInterface
private $logger;
private $defaultLocale;
private $endpoint;
private $intlIcuEnabled;

public function __construct(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint)
public function __construct(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint, bool $intlIcuEnabled = false)
{
$this->client = $client;
$this->loader = $loader;
$this->logger = $logger;
$this->defaultLocale = $defaultLocale;
$this->endpoint = $endpoint;
$this->intlIcuEnabled = $intlIcuEnabled;
}

public function __toString(): string
Expand Down Expand Up @@ -102,7 +104,12 @@ public function read(array $domains, array $locales): TranslatorBag

foreach ($translations as $locale => $files) {
foreach ($files as $filename => $content) {
$translatorBag->addCatalogue($this->loader->load($content['content'], $locale, str_replace('.xliff', '', $filename)));
$domain = str_replace('.xliff', '', $filename);
if ($this->intlIcuEnabled) {
$domain .= MessageCatalogueInterface::INTL_DOMAIN_SUFFIX;
}

$translatorBag->addCatalogue($this->loader->load($content['content'], $locale, $domain));
}
}

Expand Down Expand Up @@ -149,6 +156,7 @@ private function exportFiles(array $locales, array $domains): array
'filter_langs' => array_values($locales),
'filter_filenames' => array_map([$this, 'getLokaliseFilenameFromDomain'], $domains),
'export_empty_as' => 'skip',
'placeholder_format' => $this->intlIcuEnabled ? 'icu' : 'symfony',

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
'placeholder_format' => $this->intlIcuEnabled ? 'icu' : 'symfony',
'placeholder_format' => $this->intlIcuEnabled ? 'icu' : 'symfony',
'plural_format' => $this->intlIcuEnabled ? 'icu' : 'symfony',

A different parameter is required to ensure that the plurals are also in the expected ICU format. The param is called plural_format which should contain the same value as placeholder_format. See https://developers.lokalise.com/reference/download-files for all params. It would be great if that can be included. Without this option users might get invalid ICU translation files

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without this change, you will still get plurals in the old format from Lokalise, which the Symfony Translator is not expecting in ICU files

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would be really weird behaviour IMHO, but better safe then sorry i guess :D

Could you verify this?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's weird, but I guess @TwanVermeulen is right. According to the documentation https://developers.lokalise.com/reference/api-plurals-and-placeholders#plural-format-descriptions I guess we have to pass both parameters.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can add it, but as i said - cant really verify for now as I'm not working anymore on the project which needed this patch. So is this verified that this is needed and working?

],
]);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,15 @@ final class LokaliseProviderFactory extends AbstractProviderFactory
private $logger;
private $defaultLocale;
private $loader;
private $intlIcuEnabled;

public function __construct(HttpClientInterface $client, LoggerInterface $logger, string $defaultLocale, LoaderInterface $loader)
public function __construct(HttpClientInterface $client, LoggerInterface $logger, string $defaultLocale, LoaderInterface $loader, bool $intlIcuEnabled = false)
{
$this->client = $client;
$this->logger = $logger;
$this->defaultLocale = $defaultLocale;
$this->loader = $loader;
$this->intlIcuEnabled = $intlIcuEnabled;
}

/**
Expand All @@ -58,7 +60,7 @@ public function create(Dsn $dsn): ProviderInterface
],
]);

return new LokaliseProvider($client, $this->loader, $this->logger, $this->defaultLocale, $endpoint);
return new LokaliseProvider($client, $this->loader, $this->logger, $this->defaultLocale, $endpoint, $this->intlIcuEnabled);
}

protected function getSupportedSchemes(): array
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

use Symfony\Component\HttpClient\MockHttpClient;
use Symfony\Component\HttpClient\Response\MockResponse;
use Symfony\Component\Translation\Bridge\Lokalise\LokaliseProvider;
use Symfony\Component\Translation\Bridge\Lokalise\LokaliseProviderFactory;
use Symfony\Component\Translation\Provider\Dsn;
use Symfony\Component\Translation\Provider\ProviderFactoryInterface;
Expand Down Expand Up @@ -57,6 +58,16 @@ public function testBaseUri()
$this->assertMatchesRegularExpression('/https:\/\/api.lokalise.com\/api2\/projects\/PROJECT_ID\/*/', $response->getRequestUrl());
}

public function testIntlIcuEnabled()
{
$response = new MockResponse(json_encode(['files' => []]));
$httpClient = new MockHttpClient([$response]);
$factory = new LokaliseProviderFactory($httpClient, $this->getLogger(), $this->getDefaultLocale(), $this->getLoader(), true);
$provider = $factory->create(new Dsn('lokalise://PROJECT_ID:API_KEY@default'));

$this->assertInstanceOf(LokaliseProvider::class, $provider);
}

public function createFactory(): ProviderFactoryInterface
{
return new LokaliseProviderFactory($this->getClient(), $this->getLogger(), $this->getDefaultLocale(), $this->getLoader());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
use Symfony\Component\Translation\Loader\LoaderInterface;
use Symfony\Component\Translation\Loader\XliffFileLoader;
use Symfony\Component\Translation\MessageCatalogue;
use Symfony\Component\Translation\MessageCatalogueInterface;
use Symfony\Component\Translation\Provider\ProviderInterface;
use Symfony\Component\Translation\Test\ProviderTestCase;
use Symfony\Component\Translation\TranslatorBag;
Expand All @@ -27,9 +28,9 @@

class LokaliseProviderTest extends ProviderTestCase
{
public function createProvider(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint): ProviderInterface
public function createProvider(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint, bool $intlIcuEnabled = false): ProviderInterface
{
return new LokaliseProvider($client, $loader, $logger, $defaultLocale, $endpoint);
return new LokaliseProvider($client, $loader, $logger, $defaultLocale, $endpoint, $intlIcuEnabled);
}

public function toStringProvider(): iterable
Expand Down Expand Up @@ -231,7 +232,7 @@ public function testCompleteWriteProcess()
]))->withOptions([
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
'headers' => ['X-Api-Token' => 'API_KEY'],
]), $this->getLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com');
]), $this->getLoader(), $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com', false);

$translatorBag = new TranslatorBag();
$translatorBag->addCatalogue(new MessageCatalogue('en', [
Expand All @@ -256,6 +257,7 @@ public function testReadForOneLocaleAndOneDomain(string $locale, string $domain,
$expectedBody = json_encode([
'format' => 'symfony_xliff',
'original_filenames' => true,
'placeholder_format' => 'symfony',
'directory_prefix' => '%LANG_ISO%',
'filter_langs' => [$locale],
'filter_filenames' => [$domain.'.xliff'],
Expand Down Expand Up @@ -601,4 +603,119 @@ public function getResponsesForManyLocalesAndManyDomains(): \Generator
$expectedTranslatorBag,
];
}

/**
* @dataProvider getResponsesForOneLocaleAndOneDomainWithIntlIcuEnabled
*/
public function testReadForOneLocaleAndOneDomainWithIntlIcuEnabled(string $locale, string $domain, string $responseContent, TranslatorBag $expectedTranslatorBag)
{
$response = function (string $method, string $url, array $options = []) use ($locale, $domain, $responseContent): ResponseInterface {
$expectedBody = json_encode([
'format' => 'symfony_xliff',
'original_filenames' => true,
'placeholder_format' => 'icu',
'directory_prefix' => '%LANG_ISO%',
'filter_langs' => [$locale],
'filter_filenames' => [$domain.'.xliff'],
'export_empty_as' => 'skip',
]);

$this->assertSame('POST', $method);
$this->assertSame('https://api.lokalise.com/api2/projects/PROJECT_ID/files/export', $url);
$this->assertJsonStringEqualsJsonString($expectedBody, $options['body']);

return new MockResponse(json_encode([
'files' => [
$locale => [
$domain.'.xliff' => [
'content' => $responseContent,
],
],
],
]));
};

$loader = $this->getLoader();
$loader->expects($this->once())
->method('load')
->willReturn((new XliffFileLoader())->load($responseContent, $locale, $domain.MessageCatalogueInterface::INTL_DOMAIN_SUFFIX));

$provider = $this->createProvider((new MockHttpClient($response))->withOptions([
'base_uri' => 'https://api.lokalise.com/api2/projects/PROJECT_ID/',
'headers' => ['X-Api-Token' => 'API_KEY'],
]), $loader, $this->getLogger(), $this->getDefaultLocale(), 'api.lokalise.com', true);
$translatorBag = $provider->read([$domain], [$locale]);

// We don't want to assert equality of metadata here, due to the ArrayLoader usage.
foreach ($translatorBag->getCatalogues() as $catalogue) {
$catalogue->deleteMetadata('', '');
}

$this->assertEquals($expectedTranslatorBag->getCatalogues(), $translatorBag->getCatalogues());
}

public function getResponsesForOneLocaleAndOneDomainWithIntlIcuEnabled(): \Generator
{
$arrayLoader = new ArrayLoader();

$expectedTranslatorBagEn = new TranslatorBag();
$expectedTranslatorBagEn->addCatalogue($arrayLoader->load([
'index.hello' => 'Hello',
'index.greetings' => 'Welcome, {firstname}!',
], 'en', 'messages'.MessageCatalogueInterface::INTL_DOMAIN_SUFFIX));

yield ['en', 'messages', <<<'XLIFF'
<?xml version="1.0" encoding="UTF-8"?>
<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 http://docs.oasis-open.org/xliff/v1.2/os/xliff-core-1.2-strict.xsd">
<file original="" datatype="plaintext" xml:space="preserve" source-language="en" target-language="en">
<header>
<tool tool-id="lokalise.com" tool-name="Lokalise"/>
</header>
<body>
<trans-unit id="index.greetings" resname="index.greetings">
<source>index.greetings</source>
<target>Welcome, {firstname}!</target>
</trans-unit>
<trans-unit id="index.hello" resname="index.hello">
<source>index.hello</source>
<target>Hello</target>
</trans-unit>
</body>
</file>
</xliff>
XLIFF
,
$expectedTranslatorBagEn,
];

$expectedTranslatorBagFr = new TranslatorBag();
$expectedTranslatorBagFr->addCatalogue($arrayLoader->load([
'index.hello' => 'Bonjour',
'index.greetings' => 'Bienvenue, {firstname} !',
], 'fr', 'messages'.MessageCatalogueInterface::INTL_DOMAIN_SUFFIX));

yield ['fr', 'messages', <<<'XLIFF'
<?xml version="1.0" encoding="UTF-8"?>
<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.2" xsi:schemaLocation="urn:oasis:names:tc:xliff:document:1.2 http://docs.oasis-open.org/xliff/v1.2/os/xliff-core-1.2-strict.xsd">
<file original="" datatype="plaintext" xml:space="preserve" source-language="en" target-language="fr">
<header>
<tool tool-id="lokalise.com" tool-name="Lokalise"/>
</header>
<body>
<trans-unit id="index.greetings" resname="index.greetings">
<source>index.greetings</source>
<target>Bienvenue, {firstname} !</target>
</trans-unit>
<trans-unit id="index.hello" resname="index.hello">
<source>index.hello</source>
<target>Bonjour</target>
</trans-unit>
</body>
</file>
</xliff>
XLIFF
,
$expectedTranslatorBagFr,
];
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ abstract class ProviderTestCase extends TestCase
protected $loader;
protected $xliffFileDumper;

abstract public function createProvider(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint): ProviderInterface;
abstract public function createProvider(HttpClientInterface $client, LoaderInterface $loader, LoggerInterface $logger, string $defaultLocale, string $endpoint, bool $intlIcuEnabled = false): ProviderInterface;

/**
* @return iterable<array{0: string, 1: ProviderInterface}>
Expand Down
0