10000 feature #38182 [HttpClient] Added RetryHttpClient (jderusse) · symfony/symfony@8d45013 · GitHub
[go: up one dir, main page]
Skip to content

Commit 8d45013

Browse files
fabpotfabpot
committed
feature #38182 [HttpClient] Added RetryHttpClient (jderusse)
This PR was squashed before being merged into the 5.2-dev branch. Discussion ---------- [HttpClient] Added RetryHttpClient | Q | A | ------------- | --- | Branch? | master | Bug fix? | no | New feature? | yes | Deprecations? | no | Tickets | / | License | MIT | Doc PR | TODO This PR adds a new HttpClient decorator to automatically retry failed requests. When calling API, A very small % of requests are expected to timeout due to transient network issues. Some providers like AWS recommends retrying these requests and use a lower connection timeout so that the clients can fail fast and retry. I used the almost the same configuration as Messenger ```yaml framework: http_client: default_options: retry_failed: enabled: true // default false decider_service: null backoff_service: null http_codes: [423, 425, 429, 500, 502, 503, 504, 507, 510] max_retries: 3 delay: 1000 multiplier: 2 max_delay: 0 response_header: true scoped_clients: github: scope: 'https://api\.github\.com' retry_failed: max_delay: 2000 ``` Commits ------- 712ac59 [HttpClient] Added RetryHttpClient
2 parents 86c7113 + 712ac59 commit 8d45013

19 files changed

+662
-6
lines changed

‎src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ CHANGELOG
1010
`cache_clearer`, `filesystem` and `validator` services to private.
1111
* Added `TemplateAwareDataCollectorInterface` and `AbstractDataCollector` to simplify custom data collector creation and leverage autoconfiguration
1212
* Add `cache.adapter.redis_tag_aware` tag to use `RedisCacheAwareAdapter`
13+
* added `framework.http_client.retry_failing` configuration tree
1314

1415
5.1.0
1516
-----

‎src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
use Symfony\Bundle\FullStack;
1818
use Symfony\Component\Asset\Package;
1919
use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
20+
use Symfony\Component\Config\Definition\Builder\NodeBuilder;
2021
use Symfony\Component\Config\Definition\Builder\TreeBuilder;
2122
use Symfony\Component\Config\Definition\ConfigurationInterface;
2223
use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException;
@@ -1369,6 +1370,25 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
13691370
->info('HTTP Client configuration')
13701371
->{!class_exists(FullStack::class) && class_exists(HttpClient::class) ? 'canBeDisabled' : 'canBeEnabled'}()
13711372
->fixXmlConfig('scoped_client')
1373+
->beforeNormalization()
1374+
->always(function ($config) {
1375+
if (empty($config['scoped_clients']) || !\is_array($config['default_options']['retry_failed'] ?? null)) {
1376+
return $config;
1377+
}
1378+
1379+
foreach ($config['scoped_clients'] as &$scopedConfig) {
1380+
if (!isset($scopedConfig['retry_failed']) || true === $scopedConfig['retry_failed']) {
1381+
$scopedConfig['retry_failed'] = $config['default_options']['retry_failed'];
1382+
continue;
1383+
}
1384+
if (\is_array($scopedConfig['retry_failed'])) {
1385+
$scopedConfig['retry_failed'] = $scopedConfig['retry_failed'] + $config['default_options']['retry_failed'];
1386+
}
1387+
}
1388+
1389+
return $config;
1390+
})
1391+
->end()
13721392
->children()
13731393
->integerNode('max_host_connections')
13741394
->info('The maximum number of connections to a single host.')
@@ -1454,6 +1474,7 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
14541474
->variableNode('md5')->end()
14551475
->end()
14561476
->end()
1477+
->append($this->addHttpClientRetrySection())
14571478
->end()
14581479
->end()
14591480
->scalarNode('mock_response_factory')
@@ -1596,6 +1617,7 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
15961617
->variableNode('md5')->end()
15971618
->end()
15981619
->end()
1620+
->append($this->addHttpClientRetrySection())
15991621
->end()
16001622
->end()
16011623
->end()
@@ -1605,6 +1627,50 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
16051627
;
16061628
}
16071629

1630+
private function addHttpClientRetrySection()
1631+
{
1632+
$root = new NodeBuilder();
1633+
1634+
return $root
1635+
->arrayNode('retry_failed')
1636+
->fixXmlConfig('http_code')
1637+
->canBeEnabled()
1638+
->addDefaultsIfNotSet()
1639+
->beforeNormalization()
1640+
->always(function ($v) {
1641+
if (isset($v['backoff_service']) && (isset($v['delay']) || isset($v['multiplier']) || isset($v['max_delay']))) {
1642+
throw new \InvalidArgumentException('The "backoff_service" option cannot be used along with the "delay", "multiplier" or "max_delay" options.');
1643+< F438 div class="diff-text-inner"> }
1644+
if (isset($v['decider_service']) && (isset($v['http_codes']))) {
1645+
throw new \InvalidArgumentException('The "decider_service" option cannot be used along with the "http_codes" options.');
1646+
}
1647+
1648+
return $v;
1649+
})
1650+
->end()
1651+
->children()
1652+
->scalarNode('backoff_service')->defaultNull()->info('service id to override the retry backoff')->end()
1653+
->scalarNode('decider_service')->defaultNull()->info('service id to override the retry decider')->end()
1654+
->arrayNode('http_codes')
1655+
->performNoDeepMerging()
1656+
->beforeNormalization()
1657+
->ifArray()
1658+
->then(function ($v) {
1659+
return array_filter(array_values($v));
1660+
})
1661+
->end()
1662+
->prototype('integer')->end()
1663+
->info('A list of HTTP status code that triggers a retry')
1664+
->defaultValue([423, 425, 429, 500, 502, 503, 504, 507, 510])
1665+
->end()
1666+
->integerNode('max_retries')->defaultValue(3)->min(0)->end()
1667+
->integerNode('delay')->defaultValue(1000)->min(0)->info('Time in ms to delay (or the initial value when multiplier is used)')->end()
1668+
->floatNode('multiplier')->defaultValue(2)->min(1)->info('If greater than 1, delay will grow exponentially for each retry: (delay * (multiple ^ retries))')->end()
1669+
->integerNode('max_delay')->defaultValue(0)->min(0)->info('Max time in ms that a retry should ever be delayed (0 = infinite)')->end()
1670+
->end()
1671+
;
1672+
}
1673+
16081674
private function addMailerSection(ArrayNodeDefinition $rootNode)
16091675
{
16101676
$rootNode

‎src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
use Symfony\Component\Form\FormTypeGuesserInterface;
6565
use Symfony\Component\Form\FormTypeInterface;
6666
use Symfony\Component\HttpClient\MockHttpClient;
67+
use Symfony\Component\HttpClient\RetryableHttpClient;
6768
use Symfony\Component\HttpClient\ScopingHttpClient;
6869
use Symfony\Component\HttpFoundation\Request;
6970
use Symfony\Component\HttpKernel\CacheClearer\CacheClearerInterface;
@@ -1991,7 +1992,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
19911992
{
19921993
$loader->load('http_client.php');
19931994

1994-
$container->getDefinition('http_client')->setArguments([$config['default_options'] ?? [], $config['max_host_connections'] ?? 6]);
1995+
$options = $config['default_options'] ?? [];
1996+
$retryOptions = $options['retry_failed'] ?? ['enabled' => false];
1997+
unset($options['retry_failed']);
1998+
$container->getDefinition('http_client')->setArguments([$options, $config['max_host_connections'] ?? 6]);
19951999

19962000
if (!$hasPsr18 = interface_exists(ClientInterface::class)) {
19972001
$container->removeDefinition('psr18.http_client');
@@ -2002,15 +2006,20 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
20022006
$container->removeDefinition(HttpClient::class);
20032007
}
20042008

2005-
$httpClientId = $this->isConfigEnabled($container, $profilerConfig) ? '.debug.http_client.inner' : 'http_client';
2009+
if ($this->isConfigEnabled($container, $retryOptions)) {
2010+
$this->registerHttpClientRetry($retryOptions, 'http_client', $container);
2011+
}
20062012

2013+
$httpClientId = $retryOptions['enabled'] ?? false ? 'http_client.retry.inner' : ($this->isConfigEnabled($container, $profilerConfig) ? '.debug.http_client.inner' : 'http_client');
20072014
foreach ($config['scoped_clients'] as $name => $scopeConfig) {
20082015
if ('http_client' === $name) {
20092016
throw new InvalidArgumentException(sprintf('Invalid scope name: "%s" is reserved.', $name));
20102017
}
20112018

20122019
$scope = $scopeConfig['scope'] ?? null;
20132020
unset($scopeConfig['scope']);
2021+
$retryOptions = $scopeConfig['retry_failed'] ?? ['enabled' => false];
2022+
unset($scopeConfig['retry_failed']);
20142023

20152024
if (null === $scope) {
20162025
$baseUri = $scopeConfig['base_uri'];
@@ -2028,6 +2037,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
20282037
;
20292038
}
20302039

2040+
if ($this->isConfigEnabled($container, $retryOptions)) {
2041+
$this->registerHttpClientRetry($retryOptions, $name, $container);
2042+
}
2043+
20312044
$container->registerAliasForArgument($name, HttpClientInterface::class);
20322045

20332046
if ($hasPsr18) {
@@ -2045,6 +2058,44 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
20452058
}
20462059
}
20472060

2061+
private function registerHttpClientRetry(array $retryOptions, string $name, ContainerBuilder $container)
2062+
{
2063+
if (!class_exists(RetryableHttpClient::class)) {
2064+
throw new LogicException('Retry failed request support cannot be enabled as version 5.2+ of the HTTP Client component is required.');
2065+
}
2066+
2067+
if (null !== $retryOptions['backoff_service']) {
2068+
$backoffReference = new Reference($retryOptions['backoff_service']);
2069+
} else {
2070+
$retryServiceId = $name.'.retry.exponential_backoff';
2071+
$retryDefinition = new ChildDefinition('http_client.retry.abstract_exponential_backoff');
2072+
$retryDefinition
2073+
->replaceArgument(0, $retryOptions['delay'])
2074+
->replaceArgument(1, $retryOptions['multiplier'])
2075+
->replaceArgument(2, $retryOptions['max_delay']);
2076+
$container->setDefinition($retryServiceId, $retryDefinition);
2077+
2078+
$backoffReference = new Reference($retryServiceId);
2079+
}
2080+
if (null !== $retryOptions['decider_service']) {
2081+
$deciderReference = new Reference($retryOptions['decider_service']);
2082+
} else {
2083+
$retryServiceId = $name.'.retry.decider';
2084+
$retryDefinition = new ChildDefinition('http_client.retry.abstract_httpstatuscode_decider');
2085+
$retryDefinition
2086+
->replaceArgument(0, $retryOptions['http_codes']);
2087+
$container->setDefinition($retryServiceId, $retryDefinition);
2088+
2089+
$deciderReference = new Reference($retryServiceId);
2090+
}
2091+
2092+
$container
2093+
->register($name.'.retry', RetryableHttpClient::class)
2094+
->setDecoratedService($name)
2095+
->setArguments([new Reference($name.'.retry.inner'), $deciderReference, $backoffReference, $retryOptions['max_retries'], new Reference('logger')])
2096+
->addTag('monolog.logger', ['channel' => 'http_client']);
2097+
}
2098+
20482099
private function registerMailerConfiguration(array $config, ContainerBuilder $container, PhpFileLoader $loader)
20492100
{
20502101
if (!class_exists(Mailer::class)) {

‎src/Symfony/Bundle/FrameworkBundle/Resources/config/http_client.php

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
use Symfony\Component\HttpClient\HttpClient;
1818
use Symfony\Component\HttpClient\HttplugClient;
1919
use Symfony\Component\HttpClient\Psr18Client;
20+
use Symfony\Component\HttpClient\Retry\ExponentialBackOff;
21+
use Symfony\Component\HttpClient\Retry\HttpStatusCodeDecider;
2022
use Symfony\Contracts\HttpClient\HttpClientInterface;
2123

2224
return static function (ContainerConfigurator $container) {
@@ -48,5 +50,19 @@
4850
service(ResponseFactoryInterface::class)->ignoreOnInvalid(),
4951
service(StreamFactoryInterface::class)->ignoreOnInvalid(),
5052
])
53+
54+
// retry
55+
->set('http_client.retry.abstract_exponential_backoff', ExponentialBackOff::class)
56+
->abstract()
57+
->args([
58+
abstract_arg('delay ms'),
59+
abstract_arg('multiplier'),
60+
abstract_arg('max delay ms'),
61+
])
62+
->set('http_client.retry.abstract_httpstatuscode_decider', HttpStatusCodeDecider::class)
63+
->abstract()
64+
->args([
65+
abstract_arg('http codes'),
66+
])
5167
;
5268
};

‎src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -520,6 +520,7 @@
520520
<xsd:element name="resolve" type="http_resolve" minOccurs="0" maxOccurs="unbounded" />
521521
<xsd:element name="header" type="http_header" minOccurs="0" maxOccurs="unbounded" />
522522
<xsd:element name="peer-fingerprint" type="fingerprint" minOccurs="0" maxOccurs="unbounded" />
523+
<xsd:element name="retry-failed" type="http_client_retry_failed" minOccurs="0" maxOccurs="1" />
523524
</xsd:choice>
524525
<xsd:attribute name="max-redirects" type="xsd:integer" />
525526
<xsd:attribute name="http-version" type="xsd:string" />
@@ -536,7 +537,6 @@
536537
<xsd:attribute name="local-pk" type="xsd:string" />
537538
<xsd:attribute name="passphrase" type="xsd:string" />
538539
<xsd:attribute name="ciphers" type="xsd:string" />
539-
540540
</xsd:complexType>
541541

542542
<xsd:complexType name="http_client_scope_options" mixed="true">
@@ -545,6 +545,7 @@
545545
<xsd:element name="resolve" type="http_resolve" minOccurs="0" maxOccurs="unbounded" />
546546
<xsd:element name="header" type="http_header" minOccurs="0" maxOccurs="unbounded" />
547547
<xsd:element name="peer-fingerprint" type="fingerprint" minOccurs="0" maxOccurs="unbounded" />
548+
<xsd:element name="retry-failed" type="http_client_retry_failed" minOccurs="0" maxOccurs="1" />
548549
</xsd:choice>
549550
<xsd:attribute name="name" type="xsd:string" />
550551
<xsd:attribute name="scope" type="xsd:string" />
@@ -575,6 +576,20 @@
575576
</xsd:choice>
576577
</xsd:complexType>
577578

579+
<xsd:complexType name="http_client_retry_failed">
580+
<xsd:sequence>
581+
<xsd:element name="http-code" type="xsd:integer" minOccurs="0" maxOccurs="unbounded" />
582+
</xsd:sequence>
583+
<xsd:attribute name="enabled" type="xsd:boolean" />
584+
<xsd:attribute name="backoff-service" type="xsd:string" />
585+
<xsd:attribute name="decider-service" type="xsd:string" />
586+
<xsd:attribute name="max-retries" type="xsd:integer" />
587+
<xsd:attribute name="delay" type="xsd:integer" />
588+
<xsd:attribute name="multiplier" type="xsd:float" />
589+
<xsd:attribute name="max-delay" type="xsd:float" />
590+
<xsd:attribute name="response_header" type="xsd:boolean" />
591+
</xsd:complexType>
592+
578593
<xsd:complexType name="http_query" mixed="true">
579594
<xsd:attribute name="key" type="xsd:string" />
580595
</xsd:complexType>

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/php/http_client_retry.php

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?php
2+
3+
$container->loadFromExtension('framework', [
4+
'http_client' => [
5+
'default_options' => [
6+
'retry_failed' => [
7+
'backoff_service' => null,
8+
'decider_service' => null,
9+
'http_codes' => [429, 500],
10+
'max_retries' => 2,
11+
'delay' => 100,
12+
'multiplier' => 2,
13+
'max_delay' => 0,
14+
]
15+
],
16+
'scoped_clients' => [
17+
'foo' => [
18+
'base_uri' => 'http://example.com',
19+
'retry_failed' => ['multiplier' => 4],
20+
],
21+
],
22+
],
23+
]);

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/xml/http_client_retry.xml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<container xmlns="http://symfony.com/schema/dic/services"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xmlns:framework="http://symfony.com/schema/dic/symfony"
5+
xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd
6+
http://symfony.com/schema/dic/symfony http://symfony.com/schema/dic/symfony/symfony-1.0.xsd">
7+
8+
<framework:config>
9+
<framework:http-client>
10+
<framework:default-options>
11+
<framework:retry-failed
12+
delay="100"
13+
max-delay="0"
14+
max-retries="2"
15+
multiplier="2">
16+
<framework:http-code>429</framework:http-code>
17+
<framework:http-code>500</framework:http-code>
18+
</framework:retry-failed>
19+
</framework:default-options>
20+
<framework:scoped-client name="foo" base-uri="http://example.com">
21+
<framework:retry-failed multiplier="4"/>
22+
</framework:scoped-client>
23+
</framework:http-client>
24+
</framework:config>
25+
</container>

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/yml/http_client_retry.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
framework:
2+
http_client:
3+
default_options:
4+
retry_failed:
5+
backoff_service: null
6+
decider_service: null
7+
http_codes: [429, 500]
8+
max_retries: 2
9+
delay: 100
10+
multiplier: 2
11+
max_delay: 0
12+
scoped_clients:
13+
foo:
14+
base_uri: http://example.com
15+
retry_failed:
16+
multiplier: 4

‎src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTest.php

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,13 @@
3636
use Symfony\Component\DependencyInjection\ContainerBuilder;
3737
use Symfony\Component\DependencyInjection\ContainerInterface;
3838
use Symfony\Component\DependencyInjection\Definition;
39+
use Symfony\Component\DependencyInjection\Exception\LogicException;
3940
use Symfony\Component\DependencyInjection\Loader\ClosureLoader;
4041
use Symfony\Component\DependencyInjection\ParameterBag\EnvPlaceholderParameterBag;
4142
use Symfony\Component\DependencyInjection\Reference;
4243
use Symfony\Component\EventDispatcher\EventDispatcherInterface;
4344
use Symfony\Component\HttpClient\MockHttpClient;
45+
use Symfony\Component\HttpClient\RetryableHttpClient;
4446
use Symfony\Component\HttpClient\ScopingHttpClient;
4547
use Symfony\Component\HttpKernel\DependencyInjection\LoggerPass;
4648
use Symfony\Component\Messenger\Transport\TransportFactory;
@@ -1482,6 +1484,23 @@ public function testHttpClientOverrideDefaultOptions()
14821484
$this->assertSame($expected, $container->getDefinition('foo')->getArgument(2));
14831485
}
14841486

1487+
public function testHttpClientRetry()
1488+
{
1489+
if (!class_exists(RetryableHttpClient::class)) {
1490+
$this->expectException(LogicException::class);
1491+
}
1492+
$container = $this->createContainerFromFile('http_client_retry');
1493+
1494+
$this->assertSame([429, 500], $container->getDefinition('http_client.retry.decider')->getArgument(0));
1495+
$this->assertSame(100, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(0));
1496+
$this->assertSame(2, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(1));
1497+
$this->assertSame(0, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(2));
1498+
$this->assertSame(2, $container->getDefinition('http_client.retry')->getArgument(3));
1499+
1500+
$this->assertSame(RetryableHttpClient::class, $container->getDefinition('foo.retry')->getClass());
1501+
$this->assertSame(4, $container->getDefinition('foo.retry.exponential_backoff')->getArgument(1));
1502+
}
1503+
14851504
public function testHttpClientWithQueryParameterKey()
14861505
{
14871506
$container = $this->createContainerFromFile('http_client_xml_key');

‎src/Symfony/Component/HttpClient/CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ CHANGELOG
1010
* added `MockResponse::getRequestMethod()` and `getRequestUrl()` to allow inspecting which request has been sent
1111
* added `EventSourceHttpClient` a Server-Sent events stream implementing the [EventSource specification](https://www.w3.org/TR/eventsource/#eventsource)
1212
* added option "extra.curl" to allow setting additional curl options in `CurlHttpClient`
13+
* added `RetryableHttpClient` to automatically retry failed HTTP requests.
1314

1415
5.1.0
1516
-----

0 commit comments

Comments
 (0)
0