diff --git a/src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md b/src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md index cc2151d6addbf..6fe797e3a3152 100644 --- a/src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md +++ b/src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md @@ -10,6 +10,7 @@ CHANGELOG * Deprecate the `router.cache_dir` config option * Add `rate_limiter` tags to rate limiter services * Add `secrets:reveal` command + * Add `rate_limiter` option to `http_client.default_options` and `http_client.scoped_clients` 7.0 --- diff --git a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php index 1a9916ed751b4..c5efbc30f6cb9 100644 --- a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php +++ b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php @@ -1715,17 +1715,32 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode, callable $e ->fixXmlConfig('scoped_client') ->beforeNormalization() ->always(function ($config) { - if (empty($config['scoped_clients']) || !\is_array($config['default_options']['retry_failed'] ?? null)) { + if (empty($config['scoped_clients'])) { + return $config; + } + + $hasDefaultRateLimiter = isset($config['default_options']['rate_limiter']); + $hasDefaultRetryFailed = \is_array($config['default_options']['retry_failed'] ?? null); + + if (!$hasDefaultRateLimiter && !$hasDefaultRetryFailed) { return $config; } foreach ($config['scoped_clients'] as &$scopedConfig) { - if (!isset($scopedConfig['retry_failed']) || true === $scopedConfig['retry_failed']) { - $scopedConfig['retry_failed'] = $config['default_options']['retry_failed']; - continue; + if ($hasDefaultRateLimiter) { + if (!isset($scopedConfig['rate_limiter']) || true === $scopedConfig['rate_limiter']) { + $scopedConfig['rate_limiter'] = $config['default_options']['rate_limiter']; + } elseif (false === $scopedConfig['rate_limiter']) { + $scopedConfig['rate_limiter'] = null; + } } - if (\is_array($scopedConfig['retry_failed'])) { - $scopedConfig['retry_failed'] += $config['default_options']['retry_failed']; + + if ($hasDefaultRetryFailed) { + if (!isset($scopedConfig['retry_failed']) || true === $scopedConfig['retry_failed']) { + $scopedConfig['retry_failed'] = $config['default_options']['retry_failed']; + } elseif (\is_array($scopedConfig['retry_failed'])) { + $scopedConfig['retry_failed'] += $config['default_options']['retry_failed']; + } } } @@ -1830,6 +1845,10 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode, callable $e ->normalizeKeys(false) ->variablePrototype()->end() ->end() + ->scalarNode('rate_limiter') + ->defaultNull() + ->info('Rate limiter name to use for throttling requests') + ->end() ->append($this->createHttpClientRetrySection()) ->end() ->end() @@ -1978,6 +1997,10 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode, callable $e ->normalizeKeys(false) ->variablePrototype()->end() ->end() + ->scalarNode('rate_limiter') + ->defaultNull() + ->info('Rate limiter name to use for throttling requests') + ->end() ->append($this->createHttpClientRetrySection()) ->end() ->end() diff --git a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php index 38de7421dacc2..ff525bb5e07cf 100644 --- a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php +++ b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php @@ -85,6 +85,7 @@ use Symfony\Component\HttpClient\Retry\GenericRetryStrategy; use Symfony\Component\HttpClient\RetryableHttpClient; use Symfony\Component\HttpClient\ScopingHttpClient; +use Symfony\Component\HttpClient\ThrottlingHttpClient; use Symfony\Component\HttpClient\UriTemplateHttpClient; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpKernel\Attribute\AsController; @@ -345,6 +346,7 @@ public function load(array $configs, ContainerBuilder $container): void } if ($this->readConfigEnabled('http_client', $container, $config['http_client'])) { + $this->readConfigEnabled('rate_limiter', $container, $config['rate_limiter']); // makes sure that isInitializedConfigEnabled() will work $this->registerHttpClientConfiguration($config['http_client'], $container, $loader); } @@ -2409,6 +2411,8 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder $loader->load('http_client.php'); $options = $config['default_options'] ?? []; + $rateLimiter = $options['rate_limiter'] ?? null; + unset($options['rate_limiter']); $retryOptions = $options['retry_failed'] ?? ['enabled' => false]; unset($options['retry_failed']); $defaultUriTemplateVars = $options['vars'] ?? []; @@ -2430,6 +2434,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder $container->removeAlias(HttpClient::class); } + if (null !== $rateLimiter) { + $this->registerThrottlingHttpClient($rateLimiter, 'http_client', $container); + } + if ($this->readConfigEnabled('http_client.retry_failed', $container, $retryOptions)) { $this->registerRetryableHttpClient($retryOptions, 'http_client', $container); } @@ -2451,6 +2459,8 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder $scope = $scopeConfig['scope'] ?? null; unset($scopeConfig['scope']); + $rateLimiter = $scopeConfig['rate_limiter'] ?? null; + unset($scopeConfig['rate_limiter']); $retryOptions = $scopeConfig['retry_failed'] ?? ['enabled' => false]; unset($scopeConfig['retry_failed']); @@ -2470,6 +2480,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder ; } + if (null !== $rateLimiter) { + $this->registerThrottlingHttpClient($rateLimiter, $name, $container); + } + if ($this->readConfigEnabled('http_client.scoped_clients.'.$name.'.retry_failed', $container, $retryOptions)) { $this->registerRetryableHttpClient($retryOptions, $name, $container); } @@ -2507,6 +2521,25 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder } } + private function registerThrottlingHttpClient(string $rateLimiter, string $name, ContainerBuilder $container): void + { + if (!class_exists(ThrottlingHttpClient::class)) { + throw new LogicException('Rate limiter support cannot be enabled as version 7.1+ of the HttpClient component is required.'); + } + + if (!$this->isInitializedConfigEnabled('rate_limiter')) { + throw new LogicException('Rate limiter cannot be used within HttpClient as the RateLimiter component is not enabled.'); + } + + $container->register($name.'.throttling.limiter', LimiterInterface::class) + ->setFactory([new Reference('limiter.'.$rateLimiter), 'create']); + + $container + ->register($name.'.throttling', ThrottlingHttpClient::class) + ->setDecoratedService($name, null, 15) // higher priority than RetryableHttpClient (10) + ->setArguments([new Reference($name.'.throttling.inner'), new Reference($name.'.throttling.limiter')]); + } + private function registerRetryableHttpClient(array $options, string $name, ContainerBuilder $container): void { if (null !== $options['retry_strategy']) { diff --git a/src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd b/src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd index 9279eaf68755b..8ddadcc7894d4 100644 --- a/src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd +++ b/src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd @@ -661,6 +661,7 @@ + @@ -691,6 +692,7 @@ + diff --git a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/ConfigurationTest.php b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/ConfigurationTest.php index d56cfa90d7f48..0c80a6f105ee0 100644 --- a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/ConfigurationTest.php +++ b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/ConfigurationTest.php @@ -530,6 +530,46 @@ public function testEnabledLockNeedsResources() ]); } + public function testScopedHttpClientsInheritRateLimiterAndRetryFailedConfiguration() + { + $processor = new Processor(); + $configuration = new Configuration(true); + + $config = $processor->processConfiguration($configuration, [[ + 'http_client' => [ + 'default_options' => ['rate_limiter' => 'default_limiter', 'retry_failed' => ['max_retries' => 77]], + 'scoped_clients' => [ + 'foo' => ['base_uri' => 'http://example.com'], + 'bar' => ['base_uri' => 'http://example.com', 'rate_limiter' => true, 'retry_failed' => true], + 'baz' => ['base_uri' => 'http://example.com', 'rate_limiter' => false, 'retry_failed' => false], + 'qux' => ['base_uri' => 'http://example.com', 'rate_limiter' => 'foo_limiter', 'retry_failed' => ['max_retries' => 88, 'delay' => 999]], + ], + ], + ]]); + + $scopedClients = $config['http_client']['scoped_clients']; + + $this->assertSame('default_limiter', $scopedClients['foo']['rate_limiter']); + $this->assertTrue($scopedClients['foo']['retry_failed']['enabled']); + $this->assertSame(77, $scopedClients['foo']['retry_failed']['max_retries']); + $this->assertSame(1000, $scopedClients['foo']['retry_failed']['delay']); + + $this->assertSame('default_limiter', $scopedClients['bar']['rate_limiter']); + $this->assertTrue($scopedClients['bar']['retry_failed']['enabled']); + $this->assertSame(77, $scopedClients['bar']['retry_failed']['max_retries']); + $this->assertSame(1000, $scopedClients['bar']['retry_failed']['delay']); + + $this->assertNull($scopedClients['baz']['rate_limiter']); + $this->assertFalse($scopedClients['baz']['retry_failed']['enabled']); + $this->assertSame(3, $scopedClients['baz']['retry_failed']['max_retries']); + $this->assertSame(1000, $scopedClients['baz']['retry_failed']['delay']); + + $this->assertSame('foo_limiter', $scopedClients['qux']['rate_limiter']); + $this->assertTrue($scopedClients['qux']['retry_failed']['enabled']); + $this->assertSame(88, $scopedClients['qux']['retry_failed']['max_retries']); + $this->assertSame(999, $scopedClients['qux']['retry_failed']['delay']); + } + protected static function getBundleDefaultConfig() { return [ diff --git a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/php/http_client_rate_limiter.php b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/php/http_client_rate_limiter.php new file mode 100644 index 0000000000000..c8256d91348d6 --- /dev/null +++ b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/php/http_client_rate_limiter.php @@ -0,0 +1,27 @@ +loadFromExtension('framework', [ + 'annotations' => false, + 'http_method_override' => false, + 'handle_all_throwables' => true, + 'php_errors' => ['log' => true], + 'rate_limiter' => [ + 'foo_limiter' => [ + 'lock_factory' => null, + 'policy' => 'token_bucket', + 'limit' => 10, + 'rate' => ['interval' => '5 seconds', 'amount' => 10], + ], + ], + 'http_client' => [ + 'default_options' => [ + 'rate_limiter' => 'default_limiter', + ], + 'scoped_clients' => [ + 'foo' => [ + 'base_uri' => 'http://example.com', + 'rate_limiter' => 'foo_limiter', + ], + ], + ], +]); diff --git a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/xml/http_client_rate_limiter.xml b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/xml/http_client_rate_limiter.xml new file mode 100644 index 0000000000000..8c9dbcdad40a5 --- /dev/null +++ b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/xml/http_client_rate_limiter.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + diff --git a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/yml/http_client_rate_limiter.yml b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/yml/http_client_rate_limiter.yml new file mode 100644 index 0000000000000..6376192b76182 --- /dev/null +++ b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/yml/http_client_rate_limiter.yml @@ -0,0 +1,19 @@ +framework: + annotations: false + http_method_override: false + handle_all_throwables: true + php_errors: + log: true + rate_limiter: + foo_limiter: + lock_factory: null + policy: token_bucket + limit: 10 + rate: { interval: '5 seconds', amount: 10 } + http_client: + default_options: + rate_limiter: default_limiter + scoped_clients: + foo: + base_uri: http://example.com + rate_limiter: foo_limiter diff --git a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTestCase.php b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTestCase.php index 3d1e9c29b54fc..7cde707c53e66 100644 --- a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTestCase.php +++ b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTestCase.php @@ -38,6 +38,7 @@ use Symfony\Component\DependencyInjection\ContainerBuilder; use Symfony\Component\DependencyInjection\ContainerInterface; use Symfony\Component\DependencyInjection\Definition; +use Symfony\Component\DependencyInjection\Exception\LogicException; use Symfony\Component\DependencyInjection\Loader\ClosureLoader; use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; use Symfony\Component\DependencyInjection\ParameterBag\EnvPlaceholderParameterBag; @@ -51,6 +52,7 @@ use Symfony\Component\HttpClient\MockHttpClient; use Symfony\Component\HttpClient\RetryableHttpClient; use Symfony\Component\HttpClient\ScopingHttpClient; +use Symfony\Component\HttpClient\ThrottlingHttpClient; use Symfony\Component\HttpFoundation\IpUtils; use Symfony\Component\HttpKernel\DependencyInjection\LoggerPass; use Symfony\Component\HttpKernel\Fragment\FragmentUriGeneratorInterface; @@ -1986,6 +1988,35 @@ public function testHttpClientFullDefaultOptions() $this->assertSame(['foo' => ['bar' => 'baz']], $defaultOptions['extra']); } + public function testHttpClientRateLimiter() + { + if (!class_exists(ThrottlingHttpClient::class)) { + $this->expectException(LogicException::class); + } + + $container = $this->createContainerFromFile('http_client_rate_limiter'); + + $this->assertTrue($container->hasDefinition('http_client.throttling')); + $definition = $container->getDefinition('http_client.throttling'); + $this->assertSame(ThrottlingHttpClient::class, $definition->getClass()); + $this->assertSame('http_client', $definition->getDecoratedService()[0]); + $this->assertCount(2, $arguments = $definition->getArguments()); + $this->assertInstanceOf(Reference::class, $arguments[0]); + $this->assertSame('http_client.throttling.inner', (string) $arguments[0]); + $this->assertInstanceOf(Reference::class, $arguments[1]); + $this->assertSame('http_client.throttling.limiter', (string) $arguments[1]); + + $this->assertTrue($container->hasDefinition('foo.throttling')); + $definition = $container->getDefinition('foo.throttling'); + $this->assertSame(ThrottlingHttpClient::class, $definition->getClass()); + $this->assertSame('foo', $definition->getDecoratedService()[0]); + $this->assertCount(2, $arguments = $definition->getArguments()); + $this->assertInstanceOf(Reference::class, $arguments[0]); + $this->assertSame('foo.throttling.inner', (string) $arguments[0]); + $this->assertInstanceOf(Reference::class, $arguments[1]); + $this->assertSame('foo.throttling.limiter', (string) $arguments[1]); + } + public static function provideMailer(): array { return [ diff --git a/src/Symfony/Component/HttpClient/CHANGELOG.md b/src/Symfony/Component/HttpClient/CHANGELOG.md index 581247bbab847..915cefcf1a612 100644 --- a/src/Symfony/Component/HttpClient/CHANGELOG.md +++ b/src/Symfony/Component/HttpClient/CHANGELOG.md @@ -7,6 +7,7 @@ CHANGELOG * Add `HttpOptions::setHeader()` to add or replace a single header * Allow mocking `start_time` info in `MockResponse` * Add `MockResponse::fromFile()` and `JsonMockResponse::fromFile()` methods to help using fixtures files + * Add `ThrottlingHttpClient` to enable limiting the number request within a certain period 7.0 --- diff --git a/src/Symfony/Component/HttpClient/Tests/ThrottlingHttpClientTest.php b/src/Symfony/Component/HttpClient/Tests/ThrottlingHttpClientTest.php new file mode 100644 index 0000000000000..b63c5bab63a3e --- /dev/null +++ b/src/Symfony/Component/HttpClient/Tests/ThrottlingHttpClientTest.php @@ -0,0 +1,57 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\HttpClient\Tests; + +use PHPUnit\Framework\TestCase; +use Symfony\Component\HttpClient\MockHttpClient; +use Symfony\Component\HttpClient\Response\MockResponse; +use Symfony\Component\HttpClient\ThrottlingHttpClient; +use Symfony\Component\RateLimiter\RateLimiterFactory; +use Symfony\Component\RateLimiter\Storage\InMemoryStorage; + +class ThrottlingHttpClientTest extends TestCase +{ + public function testThrottling() + { + $failPauseHandler = static function (float $duration) { + self::fail(sprintf('The pause handler should\'t have been called, but it was called with %f.', $duration)); + }; + + $pauseHandler = static fn (float $expectedDuration) => function (float $duration) use ($expectedDuration) { + self::assertEqualsWithDelta($expectedDuration, $duration, 1); + }; + + $rateLimiterFactory = new RateLimiterFactory([ + 'id' => 'token_bucket', + 'policy' => 'token_bucket', + 'limit' => 2, + 'rate' => ['interval' => '5 seconds', 'amount' => 2], + ], new InMemoryStorage()); + + $client = new ThrottlingHttpClient( + new MockHttpClient([ + new MockResponse('', ['http_code' => 200, 'pause_handler' => $failPauseHandler]), + new MockResponse('', ['http_code' => 200, 'pause_handler' => $failPauseHandler]), + new MockResponse('', ['http_code' => 200, 'pause_handler' => $pauseHandler(5)]), + new MockResponse('', ['http_code' => 200, 'pause_handler' => $pauseHandler(5)]), + new MockResponse('', ['http_code' => 200, 'pause_handler' => $pauseHandler(10)]), + ]), + $rateLimiterFactory->create(), + ); + + $client->request('GET', 'http://example.com/foo'); + $client->request('GET', 'http://example.com/bar'); + $client->request('GET', 'http://example.com/baz'); + $client->request('GET', 'http://example.com/qux'); + $client->request('GET', 'http://example.com/corge'); + } +} diff --git a/src/Symfony/Component/HttpClient/ThrottlingHttpClient.php b/src/Symfony/Component/HttpClient/ThrottlingHttpClient.php new file mode 100644 index 0000000000000..66fc173053771 --- /dev/null +++ b/src/Symfony/Component/HttpClient/ThrottlingHttpClient.php @@ -0,0 +1,51 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Symfony\Component\HttpClient; + +use Symfony\Component\RateLimiter\LimiterInterface; +use Symfony\Contracts\HttpClient\HttpClientInterface; +use Symfony\Contracts\HttpClient\ResponseInterface; +use Symfony\Contracts\Service\ResetInterface; + +/** + * Limits the number of requests within a certain period. + */ +class ThrottlingHttpClient implements HttpClientInterface, ResetInterface +{ + use DecoratorTrait { + reset as private traitReset; + } + + public function __construct( + HttpClientInterface $client, + private readonly LimiterInterface $rateLimiter, + ) { + $this->client = $client; + } + + public function request(string $method, string $url, array $options = []): ResponseInterface + { + $response = $this->client->request($method, $url, $options); + + if (0 < $waitDuration = $this->rateLimiter->reserve()->getWaitDuration()) { + $response->getInfo('pause_handler')($waitDuration); + } + + return $response; + } + + public function reset(): void + { + $this->traitReset(); + $this->rateLimiter->reset(); + } +} diff --git a/src/Symfony/Component/HttpClient/composer.json b/src/Symfony/Component/HttpClient/composer.json index 278a9b23601f9..5f56f21db99ba 100644 --- a/src/Symfony/Component/HttpClient/composer.json +++ b/src/Symfony/Component/HttpClient/composer.json @@ -40,6 +40,7 @@ "symfony/http-kernel": "^6.4|^7.0", "symfony/messenger": "^6.4|^7.0", "symfony/process": "^6.4|^7.0", + "symfony/rate-limiter": "^6.4|^7.0", "symfony/stopwatch": "^6.4|^7.0" }, "conflict": {