8000 bug #11920 [Intl] Integrated ICU data into Intl component #1 (webmozart) · symfony/symfony@1ce0f87 · GitHub
[go: up one dir, main page]
Skip to content
< 8000 script crossorigin="anonymous" defer="defer" type="application/javascript" src="https://github.githubassets.com/assets/sessions-1e75b15ae60a.js">

Commit 1ce0f87

Browse files
webmozartwebmozart
committed
bug #11920 [Intl] Integrated ICU data into Intl component #1 (webmozart)
This PR was merged into the 2.3 branch. Discussion ---------- [Intl] Integrated ICU data into Intl component #1 | Q | A | ------------- | --- | Bug fix? | no | New feature? | no | BC breaks? | no | Deprecations? | no | Tests pass? | yes | Fixed tickets | #11447, #10807 | License | MIT | Doc PR | - This PR is an alternative implementation to #11884. It depends on ~~#11906~~ and ~~#11907~~ being merged first (~~these are included in the diff until after a merge+rebase~~ merged+rebased now). With this PR, the ICU component becomes obsolete. The ICU data is bundled with Intl in two different formats: JSON and the binary ICU resource bundle format (version 2) readable by PHP's `\ResourceBundle` class. For a performance comparison between the two, see my [benchmark](/webmozart/json-res-benchmark). ~~The data is contained in two zip files: json.zip (2.6MB) and rb-v2.zip (3.8MB). The handler~~ ```php \Symfony\Component\Intl\Composer\ScriptHandler::decompressData() ``` ~~needs to be added as Composer hook and decompresses the data after install/update.~~ The data is included as text/binary now. Git takes care of the compression. Before this PR can be merged, I would like to find out what the performance difference between the two formats is in real applications. For that, I need benchmarks from some real-life applications which use ICU data - e.g. in forms (language drop-downs, country selectors etc.) - for both the JSON and the binary data. You can force either format to be used by hard-coding the return value of `Intl::detectDataFormat()` to `Intl::JSON` and `Intl::RB_V2` respectively. I'll also try to create some more realistic benchmarks. If JSON is not significantly slower/takes up significantly more memory than the binary format, we can drop the binary format altogether. Commits ------- be819c1 [Intl] Integrated ICU data into Intl component
2 parents e24a822 + be819c1 commit 1ce0f87

File tree

119 files changed

+6403
-2313
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

119 files changed

+6403
-2313
lines changed

‎.travis.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,6 @@ install:
3535
- sh -c 'if [ "$components" = "no" ]; then sh -c "COMPOSER_ROOT_VERSION=dev-master composer --prefer-source --dev install"; fi;'
3636

3737
script:
38-
- sh -c 'if [ "$components" = "no" ]; then sh -c "ls -d src/Symfony/*/* | parallel --gnu --keep-order '\''echo \"Running {} tests\"; phpunit --exclude-group tty,benchmark {};'\''"; fi;'
38+
- sh -c 'if [ "$components" = "no" ]; then sh -c "ls -d src/Symfony/*/* | parallel --gnu --keep-order '\''echo \"Running {} tests\"; phpunit --exclude-group tty,benchmark,intl-data {};'\''"; fi;'
3939
- sh -c 'if [ "$components" = "no" ]; then sh -c "echo "\""Running tests requiring tty"\""; phpunit --group tty"; fi;'
40-
- sh -c 'if [ "$components" = "yes" ]; then sh -c "find src/Symfony -mindepth 3 -type f -name '\''phpunit.xml.dist'\'' | sed '\''s#\(.*\)/.*#\1#'\'' | parallel --gnu --keep-order '\''echo \"Running {} tests\"; cd {}; COMPOSER_ROOT_VERSION=dev-master composer --prefer-source --dev install; phpunit --exclude-group tty,benchmark;'\''"; fi;'
40+
- sh -c 'if [ "$components" = "yes" ]; then sh -c "find src/Symfony -mindepth 3 -type f -name '\''phpunit.xml.dist'\'' | sed '\''s#\(.*\)/.*#\1#'\'' | parallel --gnu --keep-order '\''echo \"Running {} tests\"; cd {}; COMPOSER_ROOT_VERSION=dev-master composer --prefer-source --dev install; phpunit --exclude-group tty,benchmark,intl-data;'\''"; fi;'

‎composer.json

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
],
1818
"require": {
1919
"php": ">=5.3.3",
20-
"symfony/icu": "~1.0",
2120
"doctrine/common": "~2.2",
2221
"twig/twig": "~1.12",
2322
"psr/log": "~1.0"

‎phpunit.xml.dist

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
<groups>
2424
<exclude>
2525
<group>benchmark</group>
26+
<group>intl-data</group>
2627
</exclude>
2728
</groups>
2829

‎src/Symfony/Component/Form/Tests/Extension/Core/Type/LanguageTypeTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public function testCountriesAreSelectable()
3131

3232
$this->assertContains(new ChoiceView('en', 'en', 'English'), $choices, '', false, false);
3333
$this->assertContains(new ChoiceView('en_GB', 'en_GB', 'British English'), $choices, '', false, false);
34-
$this->assertContains(new ChoiceView('en_US', 'en_US', 'U.S. English'), $choices, '', false, false);
34+
$this->assertContains(new ChoiceView('en_US', 'en_US', 'American English'), $choices, '', false, false);
3535
$this->assertContains(new ChoiceView('fr', 'fr', 'French'), $choices, '', false, false);
3636
$this->assertContains(new ChoiceView('my', 'my', 'Burmese'), $choices, '', false, false);
3737
}

‎src/Symfony/Component/Intl/ResourceBundle/Compiler/BundleCompilerInterface.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Compiler/BundleCompilerInterface.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Compiler;
12+
namespace Symfony\Component\Intl\Data\Bundle\Compiler;
1313

1414
/**
1515
* Compiles a resource bundle.

‎src/Symfony/Component/Intl/ResourceBundle/Compiler/BundleCompiler.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Compiler/GenrbCompiler.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Compiler;
12+
namespace Symfony\Component\Intl\Data\Bundle\Compiler;
1313

1414
use Symfony\Component\Intl\Exception\RuntimeException;
1515

@@ -20,7 +20,7 @@
2020
*
2121
* @internal
2222
*/
23-
class BundleCompiler implements BundleCompilerInterface
23+
class GenrbCompiler implements BundleCompilerInterface
2424
{
2525
/**
2626
* @var string The path to the "genrb" executable.

‎src/Symfony/Component/Intl/ResourceBundle/Reader/BufferedBundleReader.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Reader/BufferedBundleReader.php

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Reader;
12+
namespace Symfony\Component\Intl\Data\Bundle\Reader;
1313

14-
use Symfony\Component\Intl\ResourceBundle\Util\RingBuffer;
14+
use Symfony\Component\Intl\Data\Util\RingBuffer;
1515

1616
/**
1717
* @author Bernhard Schussek <bschussek@gmail.com>
@@ -53,12 +53,4 @@ public function read($path, $locale)
5353

5454
return $this->buffer[$hash];
5555
}
56-
57-
/**
58-
* {@inheritdoc}
59-
*/
60-
public function getLocales($path)
61-
{
62-
return $this->reader->getLocales($path);
63-
}
6456
}

‎src/Symfony/Component/Intl/ResourceBundle/Reader/StructuredBundleReader.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Reader/BundleEntryReader.php

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,24 +9,24 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Reader;
12+
namespace Symfony\Component\Intl\Data\Bundle\Reader;
1313

1414
use Symfony\Component\Intl\Exception\MissingResourceException;
1515
use Symfony\Component\Intl\Exception\OutOfBoundsException;
1616
use Symfony\Component\Intl\Exception\ResourceBundleNotFoundException;
1717
use Symfony\Component\Intl\Locale;
18-
use Symfony\Component\Intl\ResourceBundle\Util\RecursiveArrayAccess;
18+
use Symfony\Component\Intl\Data\Util\RecursiveArrayAccess;
1919

2020
/**
21-
* Default implementation of {@link StructuredBundleReaderInterface}.
21+
* Default implementation of {@link BundleEntryReaderInterface}.
2222
*
2323
* @author Bernhard Schussek <bschussek@gmail.com>
2424
*
25-
* @see StructuredResourceBundleBundleReaderInterface
25+
* @see BundleEntryReaderInterface
2626
*
2727
* @internal
2828
*/
29-
class StructuredBundleReader implements StructuredBundleReaderInterface
29+
class BundleEntryReader implements BundleEntryReaderInterface
3030
{
3131
/**
3232
* @var BundleReaderInterface
@@ -138,7 +138,7 @@ public function readEntry($path, $locale, array $indices, $fallback = true)
138138
}
139139

140140
// Remember which locales we tried
141-
$testedLocales[] = $currentLocale.'.res';
141+
$testedLocales[] = $currentLocale;
142142

143143
// Check whether fallback is allowed
144144
if (!$fallback) {
@@ -162,10 +162,10 @@ public function readEntry($path, $locale, array $indices, $fallback = true)
162162
// Entry is still NULL, read error occurred. Throw an exception
163163
// containing the detailed path and locale
164164
$errorMessage = sprintf(
165-
'Couldn\'t read the indices [%s] from "%s/%s.res".',
165+
'Couldn\'t read the indices [%s] for the locale "%s" in "%s".',
166166
implode('][', $indices),
167-
$path,
168-
$locale
167+
$locale,
168+
$path
169169
);
170170

171171
// Append fallback locales, if any
@@ -174,19 +174,11 @@ public function readEntry($path, $locale, array $indices, $fallback = true)
174174
array_shift($testedLocales);
175175

176176
$errorMessage .= sprintf(
177-
' The indices also couldn\'t be found in the fallback locale(s) "%s".',
177+
' The indices also couldn\'t be found for the fallback locale(s) "%s".',
178178
implode('", "', $testedLocales)
179179
);
180180
}
181181

182182
throw new MissingResourceException($errorMessage, 0, $exception);
183183
}
184-
185-
/**
186-
* {@inheritdoc}
187-
*/
188-
public function getLocales($path)
189-
{
190-
return $this->reader->getLocales($path);
191-
}
192184
}

‎src/Symfony/Component/Intl/ResourceBundle/Reader/StructuredBundleReaderInterface.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Reader/BundleEntryReaderInterface.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Reader;
12+
namespace Symfony\Component\Intl\Data\Bundle\Reader;
1313

1414
use Symfony\Component\Intl\Exception\MissingResourceException;
1515

@@ -20,7 +20,7 @@
2020
*
2121
* @internal
2222
*/
23-
interface StructuredBundleReaderInterface extends BundleReaderInterface
23+
interface BundleEntryReaderInterface extends BundleReaderInterface
2424
{
2525
/**
2626
* Reads an entry from a resource bundle.

‎src/Symfony/Component/Intl/ResourceBundle/Reader/BundleReaderInterface.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Reader/BundleReaderInterface.php

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Reader;
12+
namespace Symfony\Component\Intl\Data\Bundle\Reader;
1313

1414
/**
1515
* Reads resource bundle files.
@@ -30,13 +30,4 @@ interface BundleReaderInterface
3030
* complex data, a scalar value otherwise.
3131
*/
3232
public function read($path, $locale);
33-
34-
/**
35-
* Reads the available locales of a resource bundle.
36-
*
37-
* @param string $path The path to the resource bundle.
38-
*
39-
* @return string[] A list of supported locale codes.
40-
*/
41-
public function getLocales($path);
4233
}

‎src/Symfony/Component/Intl/ResourceBundle/Reader/BinaryBundleReader.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Reader/IntlBundleReader.php

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Reader;
12+
namespace Symfony\Component\Intl\Data\Bundle\Reader;
1313

1414
use Symfony\Component\Intl\Exception\ResourceBundleNotFoundException;
15-
use Symfony\Component\Intl\ResourceBundle\Util\ArrayAccessibleResourceBundle;
15+
use Symfony\Component\Intl\Data\Util\ArrayAccessibleResourceBundle;
1616

1717
/**
1818
* Reads binary .res resource bundles.
@@ -21,7 +21,7 @@
2121
*
2222
* @internal
2323
*/
24-
class BinaryBundleReader implements BundleReaderInterface
24+
class IntlBundleReader implements BundleReaderInterface
2525
{
2626
/**
2727
* {@inheritdoc}
@@ -52,18 +52,4 @@ public function read($path, $locale)
5252
// which are OK for us.
5353
return new ArrayAccessibleResourceBundle($bundle);
5454
}
55-
56-
/**
57-
* {@inheritdoc}
58-
*/
59-
public function getLocales($path)
60-
{
61-
$locales = glob($path.'/*.res');
62-
63-
// Remove file extension and sort
64-
array_walk($locales, function (&$locale) { $locale = basename($locale, '.res'); });
65-
sort($locales);
66-
67-
return $locales;
68-
}
6955
}

‎src/Symfony/Component/Intl/Data/Bundle/Reader/JsonBundleReader.php

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\Component\Intl\Data\Bundle\Reader;
13+
14+
use Symfony\Component\Intl\Exception\ResourceBundleNotFoundException;
15+
use Symfony\Component\Intl\Exception\RuntimeException;
16+
17+
/**
18+
* Reads .json resource bundles.
19+
*
20+
* @author Bernhard Schussek <bschussek@gmail.com>
21+
*
22+
* @internal
23+
*/
24+
class JsonBundleReader implements BundleReaderInterface
25+
{
26+
/**
27+
* {@inheritdoc}
28+
*/
29+
public function read($path, $locale)
30+
{
31+
$fileName = $path.'/'.$locale.'.json';
32+
33+
if (!file_exists($fileName)) {
34+
throw new ResourceBundleNotFoundException(sprintf(
35+
'The resource bundle "%s/%s.json" does not exist.',
36+
$path,
37+
$locale
38+
));
39+
}
40+
41+
if (!is_file($fileName)) {
42+
throw new RuntimeException(sprintf(
43+
'The resource bundle "%s/%s.json" is not a file.',
44+
$path,
45+
$locale
46+
));
47+
}
48+
49+
$data = json_decode(file_get_contents($fileName), true);
50+
51+
if (null === $data) {
52+
throw new RuntimeException(sprintf(
53+
'The resource bundle "%s/%s.json" contains invalid JSON: %s',
54+
$path,
55+
$locale,
56+
self::getLastJsonError()
57+
));
58+
}
59+
60+
return $data;
61+
}
62+
63+
/**
64+
* @return string The last error message created by {@link json_decode()}
65+
*
66+
* @link http://de2.php.net/manual/en/function.json-last-error-msg.php#113243
67+
*/
68+
private static function getLastJsonError()
69+
{
70+
if (function_exists('json_last_error_msg')) {
71+
return json_last_error_msg();
72+
}
73+
74+
static $errors = array(
75+
JSON_ERROR_NONE => null,
76+
JSON_ERROR_DEPTH => 'Maximum stack depth exceeded',
77+
JSON_ERROR_STATE_MISMATCH => 'Underflow or the modes mismatch',
78+
JSON_ERROR_CTRL_CHAR => 'Unexpected control character found',
79+
JSON_ERROR_SYNTAX => 'Syntax error, malformed JSON',
80+
JSON_ERROR_UTF8 => 'Malformed UTF-8 characters, possibly incorrectly encoded',
81+
);
82+
83+
$error = json_last_error();
84+
85+
return array_key_exists($error, $errors)
86+
? $errors[$error]
87+
: sprintf('Unknown error (%s)', $error);
88+
}
89+
}

‎src/Symfony/Component/Intl/ResourceBundle/Reader/PhpBundleReader.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Reader/PhpBundleReader.php

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Reader;
12+
namespace Symfony\Component\Intl\Data\Bundle\Reader;
1313

1414
use Symfony\Component\Intl\Exception\ResourceBundleNotFoundException;
1515
use Symfony\Component\Intl\Exception\RuntimeException;
@@ -48,18 +48,4 @@ public function read($path, $locale)
4848

4949
return include $fileName;
5050
}
51-
52-
/**
53-
* {@inheritdoc}
54-
*/
55-
public function getLocales($path)
56-
{
57-
$locales = glob($path.'/*.php');
58-
59-
// Remove file extension and sort
60-
array_walk($locales, function (&$locale) { $locale = basename($locale, '.php'); });
61-
sort($locales);
62-
63-
return $locales;
64-
}
6551
}

‎src/Symfony/Component/Intl/ResourceBundle/Writer/BundleWriterInterface.php renamed to ‎src/Symfony/Component/Intl/Data/Bundle/Writer/BundleWriterInterface.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* file that was distributed with this source code.
1010
*/
1111

12-
namespace Symfony\Component\Intl\ResourceBundle\Writer;
12+
namespace Symfony\Component\Intl\Data\Bundle\Writer;
1313

1414
/**
1515
* Writes resource bundle files.

0 commit comments

Comments
 (0)
0